// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

#include "test/contrib_ops/qordered_test_utils.h"

#if defined(USE_CUDA)

#include <cuda.h>

#if defined(USE_CUDA)

namespace onnxruntime {
namespace test {

static void run_qordered_longformer_attention_op_test(
    const std::vector<int8_t>& gold_result,
    const std::vector<int8_t>& inputq,
    const std::vector<int8_t>& weightq,
    const std::vector<float>& bias,
    const std::vector<MLFloat16>& attention_mask,
    const std::vector<int8_t>& global_weightq,
    const std::vector<float>& global_bias,
    const std::vector<int32_t>& global_attention_mask,
    const float scale_input,
    const float scale_weight,
    const float scale_bias,
    const float scale_qkv_gemm,
    const float scale_global_weight,
    const float scale_global_gemm,
    const float scale_output,
    const int64_t batch_size,
    const int64_t sequence_len,
    const int64_t num_heads,
    const int64_t head_size,
    const int64_t window,
    int64_t input_hidden_size = 0) {
  // Needs Turing architecture
  if (NeedSkipIfCudaArchLowerThan(750) || NeedSkipIfCudaArchGreaterEqualThan(800)) {
    return;
  }

  const int64_t hidden_size = num_heads * head_size;
  if (!input_hidden_size) input_hidden_size = hidden_size;

  OpTester test_qorder("QOrderedLongformerAttention", 1, onnxruntime::kMSDomain);
  test_qorder.AddAttribute("num_heads", (int64_t)num_heads);
  test_qorder.AddAttribute("window", (int64_t)window);
  test_qorder.AddAttribute("order_input", (int64_t)ORDER_ROW);
  test_qorder.AddAttribute("order_output", (int64_t)ORDER_ROW);
  test_qorder.AddAttribute("order_weight", (int64_t)ORDER_COL);
  test_qorder.AddAttribute("order_global_weight", (int64_t)ORDER_COL);

  test_qorder.AddInput<int8_t>("input", {batch_size, sequence_len, input_hidden_size}, inputq);
  test_qorder.AddInput<float>("scale_input", {}, {scale_input}, true);
  test_qorder.AddInput<int8_t>("weight", {input_hidden_size, 3 * hidden_size}, weightq, true);  // COL major
  test_qorder.AddInput<float>("scale_weight", {}, {scale_weight}, true);
  test_qorder.AddInput<float>("bias", {3 * hidden_size}, bias, true);
  test_qorder.AddInput<float>("scale_bias", {}, {scale_bias}, true);
  test_qorder.AddInput<float>("scale_qkv_gemm", {}, {scale_qkv_gemm}, true);
  test_qorder.AddInput<MLFloat16>("mask", {batch_size, sequence_len}, attention_mask);
  test_qorder.AddInput<int8_t>("global_weight", {input_hidden_size, 3 * hidden_size}, global_weightq, true);  // COL major
  test_qorder.AddInput<float>("scale_global_weight", {}, {scale_global_weight}, true);
  test_qorder.AddInput<float>("global_bias", {3 * hidden_size}, global_bias, true);
  test_qorder.AddInput<float>("scale_global_gemm", {}, {scale_global_gemm}, true);
  test_qorder.AddInput<int32_t>("global", {batch_size, sequence_len}, global_attention_mask);
  test_qorder.AddInput<float>("scale_output", {}, {scale_output}, true);
  test_qorder.AddOutput<int8_t>("output", {batch_size, sequence_len, hidden_size}, gold_result);

  std::vector<std::unique_ptr<IExecutionProvider>> execution_providers;
  execution_providers.push_back(DefaultCudaExecutionProvider());
  test_qorder.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
}

TEST(QOrderedTest, LongformerAttention_1x128x2x16_window_32) {
  constexpr float scale_input = 1.0f / 32.0f;
  constexpr float scale_weight = 1.0f / 64.0f;
  constexpr float scale_bias = 1.0f / 8.0f;
  constexpr float scale_qkv_gemm = 1.0f / 4.0f;
  constexpr float scale_global_weight = 1.0f / 64.0f;
  constexpr float scale_global_gemm = 1.0f / 4.0f;
  constexpr float scale_output = 1.0f / 8.0f;
  constexpr int64_t batch_size = 1;
  constexpr int64_t sequence_len = 128;
  constexpr int64_t num_heads = 2;
  constexpr int64_t head_size = 16;
  constexpr int64_t window = 32;
  constexpr int64_t input_hidden_size = 0;  // same as hidden_size

  // Following code generate the input data vectors: (Keep it here in case)
  // #include <iostream>
  // template<typename T>
  // void debug_print(const T* mat, int rows, int cols, const char* name) {
  //   std::cout << "========" << name << " : " << rows << "x" << cols << " ============" << std::endl;
  //   const T* p = mat;
  //   for (int r = 0; r < rows; r++) {
  //     std::cout << (r == 0 ? "[[" : " [");
  //     for (int c = 0; c < cols; c++) {
  //       if constexpr (std::is_same<T, int8_t>::value) {
  //         std::cout << (int)*p++;
  //       } else if constexpr (std::is_same<T, MLFloat16>::value) {
  //         std::cout << p->ToFloat();
  //         p++;
  //       } else {
  //         std::cout << *p++;
  //       }
  //       if (c < cols - 1) std::cout << ", ";
  //     }
  //     std::cout << ((r < rows - 1) ? "]," : "]]") << std::endl;
  //   }
  //   std::cout << "---------------- end of " << name << "-------" << std::endl;
  // }
  // unsigned seed = 666;
  // const int64_t hidden_size = num_heads * head_size;
  // if (!input_hidden_size) input_hidden_size = hidden_size;
  // RandomValueGenerator random_gen{seed};
  // auto inputq = GenData<int8_t>({batch_size, sequence_len, hidden_size}, 1.0f, &random_gen);
  // // clear some tail
  // for (int64_t b = 0; b < batch_size; b++) {
  //   std::fill_n(&inputq[(b + 1) * sequence_len - 8], 8, 0);
  // }
  // debug_print(inputq.data(), batch_size * sequence_len, hidden_size, "inputq");
  // float scale_input = 1.0f / 32.0f;
  // auto weightq = GenData<int8_t>({3 * hidden_size, input_hidden_size}, 1.0f, &random_gen);
  // debug_print(weightq.data(), 3 * hidden_size, input_hidden_size, "weightq");
  // float scale_weight = 1.0f / 64.0f;
  // auto bias = GenData<float>({3 * hidden_size}, 1.0 / 128.0f / 16.0f, &random_gen);
  // debug_print(bias.data(), 3, hidden_size, "bias");
  // float scale_bias = 1.0f / 8.0f;
  // float scale_qkv_gemm = 1.0f / 4.0f;
  // auto attention_mask = std::vector<MLFloat16>(batch_size * sequence_len, MLFloat16(1.0f));
  // // clear some tail
  // for (int64_t b = 0; b < batch_size; b++) {
  //   std::fill_n(&attention_mask[(b + 1) * sequence_len - 8], 8, MLFloat16(0.0f));
  // }
  // debug_print(attention_mask.data(), batch_size, sequence_len, "attention_mask");
  // auto global_weightq = GenData<int8_t>({3 * hidden_size, input_hidden_size}, 1.0f, &random_gen);
  // debug_print(global_weightq.data(), 3 * hidden_size, input_hidden_size, "global_weightq");
  // float scale_global_weight = 1.0f / 64.0f;
  // auto global_bias = GenData<float>({3 * hidden_size}, 1.0 / 128.0f / 16.0f, &random_gen);
  // debug_print(global_bias.data(), 3, hidden_size, "global_bias");
  // float scale_global_gemm = 1.0f / 4.0f;
  // auto global_attention_mask = std::vector<int32_t>(batch_size * sequence_len, 0);
  // // set each batch first 16 token as global attention token
  // for (int b = 0; b < batch_size; b++) {
  //   std::fill_n(&global_attention_mask[b * sequence_len], 16, 1);
  // }
  // debug_print(global_attention_mask.data(), batch_size, sequence_len, "global_attention_mask");
  // float scale_output = 1.0f / 8.0f;

  //========inputq : 128x32 ============
  std::vector<int8_t> inputq = {
      -127, 26, -69, -11, 81, 82, -44, -97, -24, 0, -23, 109, -21, -122, 26, 70, 123, -96, -38, 3, -117, -29, -63, 43, 11, 110, -65, 2, -114, 125, -59, 31,
      74, 27, -70, 26, -42, 68, -121, -48, 6, -32, -54, 59, 104, -24, 42, -74, 31, 37, -5, -85, -37, -106, -60, 99, -123, 85, 7, -43, -1, -125, 88, -7,
      -102, 124, -64, -19, -75, -99, -8, 85, 8, 25, 78, -47, 48, -38, -86, 115, -45, -3, 16, 2, -64, -58, -45, -97, 34, 73, -50, -126, 69, 86, 45, -20,
      -15, -15, -35, 38, 87, 33, -56, 67, -32, 56, -26, -125, 112, 123, -60, -36, -39, -70, -90, 75, -25, 48, 126, 92, 0, 0, 0, 0, 0, 0, 0, 0,
      -7, -6, 59, -59, -97, 70, 114, -71, 83, -63, -112, -89, -125, -51, -28, 42, 96, -68, -13, -72, 105, -39, -117, -106, 110, -119, 30, -105, -29, 59, -126, 87,
      -113, -51, -56, -35, 100, 3, -116, 21, -122, -60, -34, 86, -30, -122, -45, -48, 8, 122, 19, -69, 71, -69, 65, -2, 126, -38, 27, -41, 20, -44, 91, 118,
      91, -2, -29, 90, 27, -25, 23, -65, 91, 42, -17, 35, 113, 46, 108, 28, 98, 61, -105, 74, -112, -125, 109, 29, -57, 12, 122, 54, -104, 35, 18, -22,
      -93, 34, 43, -66, 102, -7, 44, 63, -60, 51, 109, 105, 10, -115, -58, 66, -108, 90, -33, -70, -79, 94, -24, -52, 102, -4, -27, 21, -115, -25, 17, -76,
      9, -64, 51, -114, -93, 106, 119, -17, 82, -53, -20, -72, 123, 115, -93, -112, 35, -53, 123, -86, -28, -115, -37, 105, -82, 8, 64, 84, 9, 111, 96, -87,
      -59, 76, 46, -116, 118, -59, 19, 48, -96, 56, 9, -48, 49, -12, 101, 92, 22, -23, -81, -50, -106, -74, 94, -27, 102, 25, 14, -80, -60, 46, -71, 104,
      33, 89, -85, -11, -95, -125, 114, 47, -117, 112, -29, 17, 63, 28, 113, -53, -126, -97, -65, -100, -24, -33, -57, -64, -121, -76, 82, -91, -88, 37, -22, -10,
      30, -38, -126, -87, -56, 122, 70, -80, -59, 116, 31, -71, 57, -55, 15, -109, -46, -106, -72, 7, -80, 52, -114, 83, -23, 76, 99, 34, -36, -46, -89, -110,
      -127, 84, -95, -71, 28, 24, -8, -55, 40, -71, -9, -50, 5, -47, 15, 60, 107, -4, 109, -25, 62, 43, -24, 48, 112, 17, -71, -96, 116, 59, 120, 79,
      -118, 61, -122, 80, 50, -73, -9, -49, -36, -101, -17, -69, -53, 75, 39, 15, -57, -61, -69, -102, -71, 12, -14, -101, 58, -91, -18, -89, 50, 22, -64, -29,
      121, -127, 42, 94, -76, -31, 53, -84, 89, 0, 3, 26, -4, -126, 100, 17, 69, 109, 17, -71, 34, 16, 52, -56, -116, -78, -84, 37, 114, 80, 115, 104,
      -41, -65, 59, 3, -86, -15, -112, 25, 7, -70, 119, -128, -4, 124, -126, -72, 15, 32, 48, 36, 40, -27, 54, 34, 102, 1, 18, -27, 57, 98, -74, -93,
      -10, 29, 60, 114, -52, 67, 56, 65, -96, 68, -99, 6, 101, 76, 3, -90, 124, -123, 65, -93, -103, 90, 22, 76, 87, -5, 9, 28, 74, -115, -4, -11,
      -49, 110, -72, -95, 44, 124, -84, -113, -99, 94, -44, -91, -49, -109, 6, -94, -43, 68, 26, 123, 103, -100, -92, -59, -18, -81, -42, 101, 24, 116, 114, -77,
      -106, 51, -69, -90, -68, 46, -83, -20, 44, -60, -118, -112, 124, 24, -3, -19, 71, -18, -127, 42, 78, -26, -81, 64, -78, 21, -101, 92, -100, 90, 79, 92,
      -68, -60, 125, 0, -20, -41, 103, 124, 32, 105, -16, 46, 12, 86, -116, 77, 84, -110, 28, 126, -117, -1, 15, -45, 73, 72, 14, 117, -13, 66, 6, 83,
      -26, 107, 60, 39, -115, -113, 40, -4, -119, 8, -126, 0, -38, 87, -59, -53, 0, 107, 113, 45, -62, -69, 116, -84, -81, 91, 96, -39, -90, 53, -10, 30,
      -89, -55, -19, 10, -54, 55, -47, -2, 79, -30, -83, 44, 120, 29, -75, 48, -109, 95, 97, 0, -105, 5, 123, 109, -67, -64, 98, -12, 63, 92, 64, 43,
      -78, 105, 53, 3, -106, -107, -32, 49, 11, -39, -61, -31, -114, 4, 54, 126, 95, -85, 65, -95, 123, -78, -21, -67, 8, -88, 69, -66, -55, 12, -15, -50,
      95, 70, -30, 106, -6, 98, 94, 102, -63, -37, 48, -108, -14, 103, -76, -43, -30, 111, 67, -47, 10, 97, -33, 87, 51, 64, -59, 2, 97, -4, 87, 18,
      -14, -56, -120, 60, -77, 89, -67, 105, 83, 16, 46, -110, -121, -57, -15, -26, 64, -5, 11, -8, 14, 66, -40, -115, -26, 86, -31, -75, 88, 63, -66, 105,
      120, 74, 86, 28, -107, 88, -80, 56, -104, 115, 62, -80, 109, 25, -87, -67, 107, 76, 5, 7, 110, -13, -40, 86, -66, 21, -72, -40, 50, -4, 95, -28,
      -117, -107, -123, 124, -51, -25, 82, 108, -108, -87, -17, -101, 106, 4, -31, 110, 65, 44, -81, -37, -68, 124, 32, 116, -106, -62, -80, 68, -40, -59, 96, -35,
      -59, -2, 42, 87, 71, -10, -105, 106, -69, 60, 89, -26, -73, -113, -53, -72, 30, 47, -121, -105, -39, 110, 104, -71, 29, 77, -28, -65, -27, 7, 28, 16,
      0, -88, -46, -75, -25, 37, 109, -1, -89, -96, -127, 22, 126, -103, 56, -59, -31, -126, -25, 104, 77, -75, 87, 39, -18, -114, 116, 78, -27, 9, 45, -29,
      17, -66, 66, 6, 54, 116, 76, -16, 49, 39, -45, -81, 83, -109, 74, -21, -93, -86, 44, -100, -99, 13, 71, -27, -94, -127, -64, 30, -60, 98, -33, 9,
      -62, 37, -104, -121, -94, 30, -120, 24, 50, 39, 0, -35, 44, -31, -66, -2, 102, -5, -6, -63, 72, -70, -46, -26, 96, 29, -35, -62, 93, -108, -113, -122,
      -116, 58, -17, 113, 91, -50, -4, 124, 124, -22, 111, -5, 14, 38, 7, 69, -77, -108, 14, -47, 69, -58, -13, 125, -122, -62, 63, 57, 3, -3, -121, -5,
      121, 1, 102, 109, 80, -92, 72, -2, -93, 16, 114, 53, 121, -88, -5, -83, 12, -32, 25, -51, 82, -15, 71, 51, 94, 37, 35, -48, 92, 74, 42, -96,
      15, -27, -89, 97, -51, 112, -77, -69, 81, -34, 29, 89, -17, -3, 94, 11, 59, 2, 85, -74, -115, 29, -126, -70, -123, -51, -79, -25, -64, 124, 75, 108,
      96, 41, 7, 31, 122, -98, 18, -22, 90, 43, -77, 116, 33, -11, -6, 116, -61, -79, -44, 60, -88, 28, -27, 30, -98, 114, -68, 126, -121, 108, 31, -40,
      -117, 124, 2, -9, -98, -2, 27, -27, 92, 32, 4, -47, 31, 76, 125, 101, 120, -89, -91, -58, -117, 122, -36, 47, -72, 35, 87, 119, -97, -119, -85, 79,
      28, -47, 55, -22, -73, -8, 54, 96, -109, -68, 8, -44, -113, 116, 35, 9, 1, -95, -124, 99, 107, -104, 76, 32, -86, 6, -71, 92, 90, 69, -26, 110,
      -109, 88, -64, 34, 4, -53, 11, 1, -87, 106, 26, -119, 95, 84, -48, -94, 23, -122, 19, 1, 95, -112, -35, 71, 106, 125, -15, -76, 116, 125, -1, 111,
      47, 38, -6, 62, -75, 47, -116, -66, 102, -31, 116, 124, 63, -18, 55, 102, 122, 21, -6, -87, 18, -48, -34, -36, 73, -84, -119, -32, -102, 55, -92, -34,
      -108, -7, -41, -4, -93, -58, 12, -113, -27, -73, 36, -31, -51, 88, 34, -89, -53, 82, 116, -17, 79, -87, -117, 55, 46, 104, 18, -68, -40, 117, 82, 19,
      74, 80, -22, -51, 20, -113, -121, -91, 90, -116, -47, 8, 92, -82, -100, 56, -63, 78, 57, -89, -32, 103, 83, 46, 57, -30, -57, 87, 76, 34, 126, -28,
      105, 115, 5, 8, -106, 4, 31, -58, -35, -87, 119, 112, -104, 67, -6, 46, -60, 86, -119, 51, 47, -123, -78, 7, 22, -88, 85, 13, 80, -64, 32, 109,
      -108, -63, 110, 95, 63, -22, -28, 66, -104, -91, 61, -57, 19, -19, -112, 4, 25, 45, -38, -1, 75, 31, -47, 28, 41, -73, -118, -98, 19, -40, 73, 121,
      -43, 60, -42, 5, -16, -33, 54, 67, -98, 85, 69, 38, 53, -13, -109, -91, 68, -97, 49, 50, 125, -68, 17, 96, -10, -53, 76, 118, 6, 108, 88, -55,
      -64, 31, -59, -117, 18, -98, -99, -75, -68, -41, -27, 10, -21, 37, 102, -111, -81, 36, -30, -128, 42, -35, 53, -14, -22, -61, -37, 36, -83, 77, -14, 51,
      -86, 37, 13, 54, 105, -1, 99, -127, -80, -49, -76, 17, 110, -33, -26, 71, -47, 17, -81, 56, -47, 90, 33, -36, 14, -116, -82, 56, 60, -118, 74, 59,
      72, -92, 48, 87, -49, 3, -123, 117, 90, 101, -39, -120, -80, 104, -35, 76, 124, 55, 71, -87, -103, -83, 15, -18, 68, -23, -44, -53, -23, -88, 94, 122,
      -9, 12, 76, -89, -93, -104, -83, -105, -46, -106, 56, -49, 21, 34, 85, -101, 79, -83, -119, -33, 28, -88, -43, 12, 100, 126, 126, 70, 18, 9, -31, 87,
      -76, 72, 28, -78, 7, -35, 54, 58, -39, 116, 78, 91, -125, 70, -9, 4, 15, -36, -13, -40, 99, -66, -95, 126, -68, -121, -105, 105, -126, 29, -43, -71,
      0, -55, 11, 32, -42, 9, -57, -118, 75, 34, -70, 16, -110, 114, -12, -8, 11, 74, 22, -13, 53, -15, 67, 88, 64, -12, -94, -40, -67, -119, -126, -36,
      109, 32, -47, 111, -83, -113, 102, -75, -60, -62, -61, -112, 85, 2, 60, 30, -84, -54, -2, 22, -54, 85, 81, -122, -45, -54, 88, 100, 38, 55, -26, -41,
      -104, -109, 8, 69, -7, 14, -60, 110, -21, 87, 75, -120, 107, -87, 53, 110, 49, -75, 121, 36, 35, -44, 33, -36, -84, -35, 65, 13, 49, -21, -47, 126,
      50, 63, -85, 95, -121, 34, 118, -43, 121, 38, -123, 43, 64, 4, 122, -31, 85, 1, -13, -124, 101, -82, 95, -85, -113, -8, 97, -97, 27, -118, 19, -26,
      -37, 99, 111, -126, 15, -85, 13, -2, -112, 45, -19, -13, 122, 104, -89, 126, -78, -84, 72, -82, 126, -41, 62, 7, 22, 88, -119, 52, 22, 62, -31, -120,
      95, 103, -62, 36, 63, -1, 68, -128, 7, -69, -115, 85, -42, 116, 62, 48, -35, 95, 115, 122, -100, 118, -9, -82, -127, 2, -84, 34, 26, -17, 16, 4,
      -98, -35, 110, -108, 18, -26, 30, -87, 24, 122, 1, -87, 60, -66, 63, -85, 71, 64, 92, -81, -115, 94, 104, -44, 109, 100, -3, 95, -101, -117, -122, -59,
      -86, -2, -67, 34, -40, 10, -127, -58, -46, -14, 80, -20, -70, 117, -31, 111, 93, 29, -118, -92, -118, 65, 109, 110, -102, -5, 65, 33, -121, -63, 105, -89,
      -1, -107, 44, -119, -107, -113, 72, -69, 31, 46, 85, 73, 118, -84, -16, -67, 31, -90, 106, 28, 18, -13, -127, -82, 111, -66, 28, -126, 86, -77, -36, 105,
      52, -65, 26, 53, 118, 75, 111, 66, 120, 17, 93, -122, 108, 93, 7, -95, 4, 79, 76, 0, 94, -65, -86, 28, -23, 82, -32, -12, 61, 85, 16, 32,
      -68, 75, 14, 50, 75, -17, 42, 58, 43, -33, -32, -70, 66, 72, -82, -105, 87, -84, -40, 104, -71, -85, -2, -88, -123, -4, 31, 96, 53, 25, 89, -55,
      71, 43, 32, 74, -119, 88, 112, -127, 41, 113, -20, -102, -7, -71, 3, -3, 53, 77, -62, 58, -115, 18, 119, -92, 111, -67, -19, -24, -47, 32, 114, 5,
      123, -22, 125, 42, 110, -78, -62, 24, 61, -117, -40, 57, 77, -42, 85, 38, -109, -125, -83, 62, 105, 80, 62, 24, -102, 106, -44, 44, -59, -91, 52, 93,
      66, -108, -103, -43, -37, 64, -69, 47, -43, -60, -10, 78, -4, -3, 97, -54, -96, -16, -31, -74, -72, 53, -115, -80, -16, 63, 83, 88, 40, 24, 79, 92,
      103, 21, -61, 65, -41, -52, -10, -12, -16, -56, 40, -30, 92, -90, -118, -75, -121, -77, 117, 58, 100, -61, 115, 100, -9, 114, -36, 10, -50, -4, -2, -66,
      57, -106, -33, -34, 122, 56, -97, -82, -57, 8, 24, -95, 23, 118, -44, -46, -83, -104, 125, -98, -28, 69, 94, -18, 7, -27, 71, 28, -27, -126, -72, -102,
      109, -106, -82, 61, -115, -90, 71, 57, -39, 10, 47, 71, 0, 14, -18, -38, 56, 2, -93, -34, 21, 89, 33, -16, 29, 58, 1, -54, 118, -20, 93, 61,
      72, 11, 70, -35, -29, -94, -75, 80, -55, 9, -92, -116, -92, 108, -69, -114, -88, -8, -19, -10, -39, 71, -50, 7, 121, -48, 109, 96, -64, -23, -48, 18,
      -92, -45, -73, 29, -115, -42, -6, 98, -114, -114, -31, -45, 19, -5, -39, -128, -51, -77, -23, 33, -61, 117, 85, 75, 106, -120, -8, 111, -66, 74, 2, -23,
      35, -33, 86, -35, -104, -95, -94, 69, -110, -126, -45, 37, -97, 33, -94, 57, -90, 38, -57, -56, -44, -83, 108, -88, 27, -10, 124, 58, -71, 112, -35, 67,
      68, -36, 52, 63, 75, -100, 76, 105, -40, -86, 30, -95, 30, -58, -28, 47, 87, 61, -102, 4, 45, -110, 86, 83, -75, 40, 97, 28, 18, -80, 48, 125,
      53, -6, 89, 79, 112, -114, 48, -22, 63, 94, 29, 18, 125, 72, -56, 84, 34, 9, 125, -30, 83, -24, 110, -42, -24, 18, 124, 40, -116, -6, 15, 46,
      29, 115, 15, -68, 49, -102, 31, 48, 98, 19, 3, -37, 8, 7, -25, -110, 57, -3, -44, 71, 26, 34, -83, -125, -68, 3, 68, -39, 44, 109, 121, -36,
      -55, -50, -88, -52, -24, 8, -93, 117, 13, -91, 76, -95, -109, 2, -74, 22, 67, -117, -91, 126, -104, -33, 49, 99, 102, -117, 0, -44, 126, 79, 63, 24,
      106, -47, 62, -38, -8, 41, -3, 84, -11, 35, 35, -58, -29, 117, -109, 111, -115, 117, 76, -73, -89, -51, -41, 70, 30, 55, -55, 17, 72, 23, -92, 80,
      -127, -127, -28, 53, 40, 34, 72, 60, -87, -67, -103, -7, -99, 81, 7, 70, 73, 47, -81, 114, -53, 77, -79, -31, 42, 0, -103, 52, -89, -32, 33, 79,
      98, -68, -58, -107, -111, -9, 64, 72, -77, 13, 45, 36, 25, -22, 87, -54, -2, -14, -98, 44, 75, 53, 49, 110, -8, 26, -18, 20, -42, 8, 62, 87,
      119, 47, 56, 107, -69, -71, 42, 0, 76, 107, -116, -49, 126, 18, 37, 63, 123, 103, 106, 46, 126, 96, -51, -87, 59, 125, 18, 59, 13, -27, 49, 110,
      102, -35, -7, 8, -74, 102, 119, 114, -33, 117, -32, 106, 37, 23, 6, 125, 18, 117, 55, -108, -109, 112, 92, 71, 45, -125, 65, -30, -55, 8, -39, -91,
      -13, 44, -88, 38, 96, 13, -112, -118, -101, 30, 33, -44, -13, -57, -34, 122, -116, 121, -91, -57, 40, -33, -57, -2, 12, -2, -54, 22, 18, 87, -109, 125,
      -55, -83, 4, -11, 125, -19, 45, -94, 58, 50, -96, -127, 38, 28, 108, -83, 91, -31, -123, -16, -47, -99, -98, 4, 13, 32, 105, 77, 36, -20, 93, 101,
      89, -47, 3, 98, -92, -103, -102, 41, 22, -68, 19, 58, 41, -54, -117, 48, -123, 33, -4, 13, -48, 98, 35, -10, -76, -33, -29, 24, -66, -27, 75, 86,
      -37, -28, 97, -68, 62, 39, -89, -27, 9, 55, 49, 45, 57, -106, -103, -64, -122, 50, -89, -82, 4, 89, 37, 53, -64, -38, -109, -125, 91, -127, -93, 42,
      -79, -115, -123, 41, 11, -73, 97, 21, -120, 31, 80, 61, 106, -102, 63, 77, 0, 79, -30, -60, 31, -125, -17, 4, -28, 113, 68, 113, 11, -97, 112, 10,
      -108, -39, 5, 82, 69, -102, -28, 119, 81, 120, -25, 72, -45, -43, 80, -40, -81, 88, -80, 7, 6, 104, 97, 63, 26, -105, 126, 77, 31, -69, 42, 96,
      -86, 30, -106, 74, 67, -90, 91, -99, -103, -54, -55, -17, -48, -127, 124, -30, 3, -98, 88, 58, -56, 77, 44, -115, 27, -96, 112, 97, -70, 15, 23, 7,
      4, -61, 98, -75, 116, 43, 51, 33, -79, 50, -128, -55, 50, -63, 89, 5, -28, -122, 118, -27, -12, -8, 59, 0, 2, -57, -91, -25, -7, 50, -92, 74,
      42, 55, -102, 34, -106, 25, -74, 108, -125, -127, 119, -12, 31, -62, -38, 87, 45, -65, -75, 97, 7, -6, 68, -53, 6, 96, 88, -116, -87, -115, -3, 122,
      -68, 53, 63, 73, -88, -48, -4, 47, 60, -10, -116, -99, -111, -49, -87, -107, 30, -9, -95, 29, -9, 62, 87, 20, -77, 30, 8, -91, 56, 61, -87, -8,
      116, 122, 95, 95, 69, -59, -3, 3, 95, -73, 122, 91, -99, -106, -22, -26, -76, -2, 48, 93, -36, -116, -90, -94, -28, -47, 4, 7, -3, 42, 116, 115,
      78, 93, 121, -48, -64, 86, -35, -61, 80, 115, -102, -37, 101, 25, 110, 80, 49, -43, 6, 23, 56, 37, -76, 26, 22, -44, 42, -110, 3, 115, 84, 27,
      17, 48, -128, 60, 33, 27, -68, -25, -77, -1, -71, 39, -3, 73, -47, 44, 120, 38, -45, 37, -12, -60, 6, 21, 105, -48, 118, -115, 43, 55, 95, -105,
      -89, 96, -112, -48, 115, -56, 0, 60, 95, -13, -21, -106, -11, -62, -113, 34, -80, -13, 0, -100, 124, 92, -105, 99, 115, -26, -94, -104, 46, -84, -107, -127,
      115, 50, -84, 101, -64, 71, 27, 110, -102, -43, -55, 58, -56, -6, -28, -108, -8, -54, -41, 53, 7, 39, -105, 23, -3, -110, 69, -106, -96, 103, -42, 104,
      -118, -48, -89, -16, 89, 74, -76, -13, -61, -49, -80, -69, 7, 93, -16, 47, -100, -60, 59, -25, 94, 11, -96, -117, 125, -43, 28, -109, -60, 65, 4, -35,
      59, 82, 87, 25, -40, 6, -68, -52, -99, -127, -56, -86, -19, -120, -30, -119, 23, 84, 55, -97, -122, 12, -59, 59, 37, 52, 67, 34, -76, 54, -67, -92,
      7, 90, -111, -118, 110, 60, 89, -60, 59, 89, 101, 84, 93, -59, 103, -43, 3, -62, 120, 98, 114, 52, 107, -8, 37, 65, -86, 8, -36, -12, 70, -74,
      -63, 31, -14, -39, -91, -101, -61, -14, 93, -100, -102, -19, -37, 116, 126, -37, 4, 38, -59, 45, 73, -76, -9, 77, 107, 57, 6, 43, 45, 120, 14, 69,
      -7, 99, -105, 97, -90, -1, -98, -35, 48, -76, 41, -2, -60, 11, -93, -100, 5, -65, -76, -104, 120, -86, -20, -93, 97, 108, 43, 25, -45, 79, -98, -77,
      -112, -44, 62, -90, 110, 78, -93, -51, 80, -128, -30, 115, 97, 67, 102, -73, -30, 4, 103, -30, 21, -23, 64, -113, 55, 33, 47, 10, 40, 90, 20, 98,
      -74, 106, -23, -115, -11, -97, -5, 107, 117, -31, 98, -18, -81, -12, 102, 52, -6, 118, 11, 99, -47, 76, 41, 77, -69, -62, -128, 110, 85, -85, -105, -47,
      -37, -39, 125, -67, -31, 15, -69, -118, -9, 95, -86, -26, 43, 48, -53, -102, 78, 27, -37, 112, -88, -26, 54, -31, -23, -4, 91, 12, 115, -85, 50, -79,
      112, -84, 98, 1, -21, 80, -58, 46, 108, 69, -128, 71, 117, -66, -101, -69, -31, 28, -59, 36, -45, -67, -61, -126, -84, -35, 6, 69, 117, 58, -51, -30,
      -27, -12, -73, 15, 95, 105, -20, 21, 99, 112, 19, 30, 43, 41, -35, -92, -25, -58, -52, 23, -50, -87, 26, -113, -18, -35, -46, 22, 108, 77, -12, -46,
      1, -59, -79, -89, 9, -121, 117, 113, 76, -72, 86, -31, 63, 35, -56, -118, 75, 58, 123, -68, -29, 93, 100, 117, 81, 67, -51, -78, 69, 17, 26, 96,
      76, -20, -81, -35, -59, -114, 70, 37, -113, -72, -25, 15, 32, -103, 41, 75, 69, 8, -76, -127, -36, -41, -104, -6, 94, 126, -119, -10, -61, -56, -40, 25,
      15, -41, 100, 110, -77, 27, 117, -2, 45, 76, 19, -73, -123, -33, -38, -85, -122, 12, -40, -53, -90, 116, 3, -11, 35, -74, 68, 63, 87, -41, -108, 73,
      61, -115, 118, 92, -19, -30, -5, -69, 71, 118, -49, 42, -44, 19, 36, 97, -9, 66, 77, -77, -66, -3, 87, -68, -42, -58, -65, -128, -3, -104, 124, 111,
      -19, 38, 10, 22, -93, 22, 118, 5, 0, 0, -88, -79, -52, 82, -19, 9, -115, -60, -105, -27, -108, -89, -72, -58, -32, -90, -112, 21, 81, -44, -38, -51,
      83, 8, -53, 42, 4, -102, 113, -65, -13, -9, 122, 107, 111, -91, 33, 77, 35, 105, -15, -80, -99, 85, 5, -65, -81, -102, -126, 12, 90, 82, -121, 44,
      -17, -71, -53, 103, 104, -85, -69, 21, 75, 116, 55, 38, -11, -22, 83, -31, -33, 35, 75, 43, -60, -18, -5, -4, 97, -47, 120, -121, 81, -5, 113, 48,
      112, 51, 64, -4, 94, 112, 56, 55, 11, 58, 93, 126, 56, -42, 122, -121, 67, 60, 62, -63, 51, -33, 61, 70, -19, 14, 22, -42, 70, 54, 2, 11,
      -37, -63, 82, 24, -4, 13, 63, 50, 67, -82, 1, 36, 25, 96, -82, -58, 5, -11, -61, 121, -16, -17, 27, -63, 8, -110, 4, -44, -44, 26, 85, -18,
      -125, 80, -49, -50, -121, 20, 79, 82, 58, -39, 14, 71, -90, -106, -28, 83, 71, 47, -88, 82, -110, -52, -17, -45, -124, 26, -4, 46, 89, 53, 98, -41,
      0, 59, -82, -32, 89, 99, -65, -3, 113, -107, 60, -70, 7, -24, 66, 95, 95, -97, -55, 81, 53, -88, -5, -52, 70, 21, 61, -44, -8, -122, 18, -73,
      109, 14, 126, -49, 123, 93, -1, 34, 40, 67, 107, 90, -35, -98, 87, -123, -97, -124, 10, -110, 49, -124, 23, -111, 2, -31, 35, 76, -118, -31, -36, -14,
      -98, -67, 4, 15, -56, 49, -17, -94, 105, -23, 107, 24, 52, -58, 37, -32, -104, -61, 85, 34, 48, 30, 44, 5, 9, -25, 80, -94, -66, -44, 48, 111,
      -17, -102, -50, -101, -2, -75, 33, 59, 42, 97, -41, 119, -39, -83, -101, -56, -116, -71, -43, 0, -14, -32, 30, -52, 38, 21, -65, 12, 120, -66, -114, 5,
      -105, -87, 91, 93, 92, -15, 3, 121, 86, -74, 116, -43, -9, 8, 44, 58, 13, -49, -11, -100, 108, 107, -70, 8, 66, -32, 103, 59, 26, -41, -20, -102,
      -18, 6, 91, -107, -38, 27, 76, 0, -70, -49, -26, 100, -103, 78, 24, -27, -104, -27, -11, -72, -14, -4, -40, 95, -72, -110, -90, 66, 54, -26, -110, -17,
      9, -17, -117, 100, 49, 28, -68, 42, 111, 48, 71, -50, -27, -109, -75, -45, -23, 70, 17, -39, -14, -60, -41, -50, -31, -105, 21, -90, 118, -99, -2, -33,
      -77, 107, 63, 78, -53, 91, -101, -94, 68, -52, -40, -55, -128, 26, -7, -105, 3, -37, 30, 71, 8, 123, 10, -10, -104, 34, -9, -81, 20, -95, 24, -73,
      -91, -33, 92, -53, -84, -93, 57, 23, 27, -122, -118, 107, 109, 36, 60, -33, -44, 19, 75, -49, -120, 30, -62, -41, 15, 64, 74, -6, 84, -11, 3, -21,
      -78, -74, 5, -8, 29, -78, 67, 11, 17, -109, -93, 24, 59, 6, 3, -127, 96, 102, -114, 55, -86, -2, -10, 34, 76, -123, -14, -66, 3, -117, -53, 13,
      99, 32, 112, -46, 62, 17, 110, -109, 32, -112, 30, -85, 116, 106, 42, 17, 60, -111, 66, 17, -5, -80, 93, -42, 112, -23, -46, 61, 45, 13, -115, 33,
      -67, -52, -91, -67, 108, -21, 1, 7, -93, 123, 67, -101, -102, -109, 21, 12, 47, 81, -29, 64, -74, -50, 50, 2, -43, -98, -51, -13, -126, -19, 50, 67,
      114, 79, -106, 88, -16, -106, 53, 1, -86, 14, -35, -81, -46, -41, -43, -63, -75, 43, -45, 112, 35, 11, 94, -46, -103, -9, -21, 10, 39, -78, -41, 3,
      -57, 106, 46, 12, -87, -35, 57, -4, 123, 20, -4, 43, -66, 95, -67, -78, 84, 60, 37, 71, -37, 30, -37, 30, -12, 89, 99, 62, 89, -8, 86, -27,
      59, -86, 110, 102, -109, -21, 31, 22, -25, -9, 83, -24, 28, 71, -80, -53, -45, 67, 28, -71, -5, 33, 85, -119, 43, 60, -93, -111, 33, 48, 95, -12};

  // ========weightq (transposed) : 96x32 ============
  std::vector<int8_t> weightq = {
      -7, -114, -11, -31, -24, -5, -123, -20, -113, -102, 48, -80, 42, -99, -109, -103, 87, 119, -36, -3, -27, 104, -40, 12, 44, 81, 80, 110, 92, 122, -44, 71,
      -112, 8, -46, 114, -44, 76, -70, -56, 116, -103, -73, -3, 18, -32, -20, -53, -51, 18, -51, 111, -44, -16, 1, -36, 125, 68, -29, 52, -65, 115, -81, -61,
      -43, -4, -20, 77, -16, -9, -73, -124, -98, -86, -35, 38, -105, -18, -43, -38, -128, 61, -112, -67, 18, -85, 45, 19, 47, -42, 69, -58, -57, 11, -10, 84,
      70, 28, -39, 28, -43, -61, 78, -27, -121, 115, 104, -109, -40, 111, 2, -97, -105, -105, 9, 84, 33, -120, -77, 42, -57, 71, 59, 126, -25, -98, -32, 123,
      45, -50, 0, -38, -88, 28, -75, -100, 22, 113, 101, 58, 71, 124, -114, 83, -69, 18, 103, -38, 63, -46, -13, -44, 13, -68, 121, 23, 43, -115, -14, -67,
      -127, 33, 72, 81, -91, -63, -8, 80, -94, 43, -40, -116, -91, -53, 31, -31, 103, -127, -45, -38, 41, -1, 107, 80, 6, -48, 124, -17, 5, 6, -60, -95,
      -60, 115, 83, -123, 108, -62, 0, -75, 121, 122, -107, -75, -42, 41, -44, -66, 61, 5, 56, 23, -18, -121, 25, 86, -51, 121, -80, 98, 114, 38, -66, 78,
      -79, -89, 126, 77, 121, -18, -69, 29, -66, -100, 8, -29, -126, -46, 75, -63, 4, -50, 92, 121, -84, -25, 75, 88, 63, -117, 108, -106, 75, -67, -11, -10,
      80, -98, -27, -21, -26, -68, 107, -4, -72, -33, -7, -36, -45, 44, -46, -62, 105, -100, -91, 30, 79, -105, -63, 40, 107, 51, 93, 73, -105, -3, -95, -83,
      -31, 54, -103, -104, -33, -37, 81, -17, -126, 97, -60, -35, 61, -124, -44, 59, -40, -29, -61, -77, -90, 76, -22, 108, -18, 71, 112, -53, 98, 114, -98, 12,
      32, -51, 32, -17, 92, -47, -43, 61, -119, -127, -40, 56, 59, -114, -93, -67, 112, 111, -117, -119, 80, 29, 88, 34, -26, 75, -101, 74, 36, -128, -14, 33,
      -6, -65, 7, 40, 117, 52, 41, -35, 57, -30, -31, -38, -81, -123, 36, -100, 34, 84, -94, 102, -78, -63, -87, 32, -25, 124, 101, -82, 2, 1, -88, -112,
      -64, -84, 95, 37, 17, 105, 125, -74, -16, -123, 9, -61, -72, -34, -114, -48, 112, -42, -11, 51, -125, 57, 20, 98, 98, 61, 21, 67, -127, -96, -98, -39,
      -89, 51, 92, -8, 57, 111, -55, 77, 9, 29, -33, 110, 20, 33, -99, -106, -90, -13, -8, -33, -55, 90, 82, 120, -76, -96, 95, -36, -54, 79, -6, 50,
      117, -106, -107, -80, 125, 75, -3, 27, -104, 90, -95, -40, -39, -59, 92, 76, 103, -16, -121, 115, 82, -13, 66, 34, 19, 104, -67, -101, -102, 33, -24, 115,
      100, 106, 45, -57, 19, 84, 71, 59, 18, -125, 86, 75, 88, -67, -76, -107, -114, -50, 125, -55, -112, 0, -127, 81, 26, -9, 59, -2, -11, -71, -22, 109,
      4, -35, 115, 32, 17, -66, -99, -86, -59, -18, 94, -119, -24, -96, -53, -53, -41, 30, -6, 37, -102, 110, -32, -6, 103, 19, 117, -114, 116, 57, 100, -97,
      11, 11, -28, -56, -77, 108, 83, -107, -29, 48, 56, -80, 26, -127, 37, -116, 70, -47, -94, 104, -68, 99, -116, 117, -78, -4, 121, -5, 119, 114, -119, 85,
      -75, 89, 26, -23, -17, 2, 10, -61, -100, -114, 111, 0, 5, 55, -41, 121, -117, 111, 3, -39, 26, -90, 34, 20, -121, 51, 115, 96, 122, 28, -85, 92,
      43, 64, 94, -77, -101, 90, 99, -21, 76, 58, -7, -40, 77, -106, 12, -14, -38, 60, -37, 66, 112, 96, 29, 74, 100, 126, 122, 59, 86, -93, -128, -46,
      -28, 106, -31, 124, 36, 114, -18, 54, 4, -16, -14, -117, -108, -48, -69, -111, 83, -100, 93, 75, -118, -46, -116, -122, -34, 10, 68, 17, -40, 99, 116, 24,
      121, 43, -16, 107, -97, -107, -116, -54, 89, -59, 76, 103, 92, 44, 3, 31, 29, -25, 110, 50, 113, -20, -78, -47, -110, 67, 123, 42, -119, 79, -36, -32,
      91, 0, 50, -62, -17, 103, 117, 17, 58, -14, -79, 50, 19, 42, 50, 25, 7, -67, 121, -128, 68, 62, -2, 35, 93, -128, 45, -6, 31, -77, -2, 60,
      -32, -11, 21, 29, 44, -76, 11, 35, -31, -13, -58, -126, 36, -65, 19, -124, -48, -92, -48, -73, 29, 24, 101, 119, -58, 40, 7, 62, 74, 5, 18, -9,
      -51, -27, 38, 9, 10, 108, 107, -85, 100, -88, -33, -123, -119, 73, 86, -78, 42, -8, 8, -125, -76, -11, -42, 114, 32, 49, 48, -16, 27, 73, 20, -26,
      -69, 85, -99, 69, 36, -1, -99, -67, -69, -41, -76, 42, 92, -90, 111, -28, -113, -19, -26, 17, -112, -95, -104, 125, 100, -34, 16, 95, 37, 77, -113, 52,
      44, 37, 100, 97, 28, -61, 50, -85, -119, -70, -78, -52, -58, 7, -20, -84, 2, -13, -1, -68, 15, 25, -21, 43, 101, 83, -21, -109, -109, -37, 32, 106,
      -69, -115, 84, 50, -30, 80, -104, -83, 95, 72, 80, -60, -39, 82, -50, 94, -100, 77, -14, -16, 78, -128, 82, 3, -8, 70, -115, -104, 61, -106, -114, 125,
      94, 107, 15, -68, -84, -26, -56, 75, 80, -121, 39, 68, 64, 58, 118, 105, -101, 104, -97, 55, 104, -50, -39, -47, 77, -105, -51, 32, 39, 16, -121, -105,
      57, -97, -5, -54, 50, 63, 6, -49, -76, 120, -23, -17, -25, -17, -95, -39, 70, -54, 40, 116, 120, 48, -29, -64, -81, -91, 117, 116, 38, 14, -17, -41,
      -114, -79, -23, 44, -89, -56, 45, -14, -86, 83, -70, 46, 67, -22, 119, 73, 53, -44, -105, 23, 94, -8, -114, 22, -88, -8, 96, 3, 121, -12, -103, 75,
      -128, -92, -89, 116, -68, 121, -9, 5, 89, -58, -126, -78, -57, 77, 112, -126, 117, 3, -41, -99, -68, 94, -50, -35, -28, 37, -24, -112, -38, -6, -64, 15,
      -30, 54, 126, -22, 49, 2, -26, 118, 34, -29, -91, 29, -120, 86, -105, 102, 67, 85, -85, -95, 109, -126, 1, 75, -100, -79, 42, -116, -28, 21, 65, 45,
      115, 67, 93, 76, -127, 27, -85, 20, -46, 71, -7, 103, 68, 108, -61, -127, -80, -112, -121, -102, 43, -112, -55, 93, -82, 50, -65, -45, 117, 86, 53, 105,
      -12, 66, 19, 10, 126, -27, -97, -48, 124, -40, 62, 99, -120, -23, 61, -28, 50, 27, 126, -92, 85, 71, -123, 26, 8, -128, -7, -83, 17, -71, -66, 79,
      -127, -113, -107, 17, -99, 120, 18, 34, 126, 38, 85, 24, 84, -61, -70, 75, -35, -3, -5, 20, 9, -128, -99, 115, 29, -122, -40, -89, 67, 4, 38, 15,
      -92, 11, 46, -34, 38, 58, -65, -92, -50, 45, -113, 96, -65, -34, 98, -36, -66, -52, 51, 46, 68, -50, 89, -5, 123, -60, 68, -68, 87, 52, 55, -110,
      -51, 79, -95, 24, -62, -92, -93, -118, 36, 86, 41, -92, -80, 68, -77, -86, 79, 62, 30, 125, -75, 53, 74, 115, -68, 101, -3, 115, -113, 102, -123, 58,
      11, 39, 54, 41, 88, -60, -61, 92, -40, 23, 0, -13, -81, -73, -64, 3, -78, -73, 77, 53, 44, 34, -16, -3, 57, 121, -72, 109, 125, -41, -120, -10,
      -24, 11, -96, -92, -20, 5, -32, 41, -60, 16, 100, -39, -102, -87, 108, -12, -30, 5, -91, -55, -113, -9, 113, -119, -113, 26, 2, 124, -48, -27, 109, 62,
      68, 2, 123, -68, 112, -68, -44, 83, 39, 42, -48, 27, -85, -100, 38, 111, 96, 77, 106, 125, 37, 46, 68, 89, -14, 90, -92, 16, 5, -21, -119, 75,
      -71, 59, 4, -9, -124, -37, -15, 95, 81, 36, 63, 21, 15, 123, -21, 65, -16, 126, -107, -118, -113, 82, -77, -49, 99, -47, -44, 89, -108, 42, -88, 3,
      75, 8, 99, 33, -93, -62, 99, 32, -1, 34, 97, -60, 96, 60, 56, 108, 0, 4, -126, -19, 96, 102, 113, 7, 31, 55, 43, 110, -98, 20, -104, -126,
      -70, 45, -3, -118, 31, 28, -108, -123, 11, -64, -68, -22, 21, -117, 50, 106, 118, -37, -121, -45, -72, -84, 75, 119, 29, 114, 55, -29, -43, 121, -56, 23,
      31, -30, 42, 108, -74, 125, 54, 19, -62, -23, -43, 43, -112, 28, 87, -9, 4, 7, -31, -18, -43, -24, 21, -61, -51, 50, -34, 126, -127, -58, 4, -71,
      -92, 33, 113, 62, -122, 23, -104, -17, -111, 43, -30, -88, -93, -82, 114, -94, -45, 66, 119, -113, -8, 36, -16, -8, 47, -43, -62, 99, 49, -125, -106, -26,
      10, 61, 86, -120, -54, -125, -37, 3, 119, 80, 9, 107, 66, 29, 7, 7, -38, -68, 89, 105, -73, 123, -81, 121, -50, 126, -69, -45, 42, 53, 47, -124,
      -85, 42, 1, -22, 48, 55, 78, 20, -88, -119, 84, 19, 63, 32, 65, 73, -80, -93, -95, 76, 19, -109, 22, 65, -19, 77, 126, 104, -102, 30, 101, -61,
      -114, -89, -20, -11, 104, 78, -25, -2, -107, -106, 95, -112, -18, -65, -121, 31, -26, 47, -49, 9, -123, -37, 51, 72, 97, -25, 106, 38, -77, 52, -77, 113,
      -42, 117, -97, -89, -60, 120, 11, -32, -15, 11, -118, 118, 52, -57, 15, 95, 83, 71, -112, 51, -87, -70, -36, 83, -88, -11, -90, -35, 46, 85, -84, 43,
      119, 20, 76, 10, -50, 17, -102, 88, -19, 62, -52, 120, 97, 63, 56, 76, 19, -54, -28, 2, -102, 6, 20, 26, -71, 60, -17, 124, 38, -118, -107, 112,
      92, 101, -84, 52, -98, -83, 65, -6, -52, 72, -118, 20, 4, 119, -116, 68, -120, 15, 91, -62, 9, -38, 106, -30, -3, -44, -109, -38, 80, 66, 6, -115,
      -103, -49, -87, 10, 51, -123, -56, 7, -87, -20, 28, -55, 21, 122, 86, -94, -42, -3, -69, 99, -48, -81, -82, 63, -75, 72, 69, 21, -88, -29, 17, 50,
      7, 3, 97, 45, 67, 66, -95, 51, 20, -8, 2, -64, -91, -44, -35, -5, 81, -42, -30, -9, 17, 15, -119, -103, 17, 124, -63, -9, -70, 125, -73, -114,
      7, 32, -67, 34, -63, -28, -37, -70, -11, 55, 13, -66, -36, 2, 52, 34, 45, -91, -29, -36, 14, -54, -104, -82, -27, 2, -114, -45, 84, -33, 60, -101,
      33, -22, 83, 60, -17, -111, -14, 91, 35, 105, 78, -12, 6, 9, 89, 22, 113, 20, -76, -43, 11, -13, -98, -122, -104, -124, -22, 90, 44, -48, -53, -94,
      -103, 116, -13, -108, -57, -70, 79, -8, 0, -75, 10, 28, 114, -70, -122, 0, -30, 84, 7, 20, 16, -33, 73, 40, -17, 91, -56, -39, 73, 70, 104, 70,
      -38, -65, -74, 95, 19, -3, -1, 19, 89, 115, 2, 3, -32, 91, 49, -32, -117, -32, 66, -87, 50, -61, -32, 33, 110, -56, -47, 77, 77, -79, -9, 20,
      29, -17, 99, -19, 23, 21, -27, 30, 99, 115, 77, 12, 9, 101, -105, 61, 108, -35, -14, -16, 20, 2, -63, 79, -83, 38, 66, -68, 43, 5, 105, -5,
      -56, 96, -77, -2, 73, 100, -52, -80, -33, -112, -46, -31, 34, -82, -49, 78, -121, -125, -90, 24, 8, -104, -15, -13, -76, -57, 94, -115, 44, -122, 67, -92,
      -38, 106, 69, 28, -21, 39, -53, -101, 123, 80, 108, 9, 81, 117, 102, 124, -97, -48, 56, 22, 72, -46, 59, 95, -77, 126, 78, 23, 88, 54, 18, -23,
      -45, 91, -101, -52, 41, 84, 83, 68, -124, 92, 125, 10, -55, 93, 113, 42, 100, -27, -40, -30, 5, -35, 92, 84, -93, 69, 30, 62, 57, 26, 43, 24,
      -125, 118, 42, 81, -115, -88, 101, 85, 8, -87, 50, 35, 29, -104, -26, -54, 70, 42, 40, 106, -19, -95, 82, -7, 114, 31, -35, 82, 32, 75, 106, -82,
      -22, 119, 28, -93, 101, -42, 89, -39, -125, 15, 53, -127, 10, 43, -127, 3, -115, -24, 108, -3, -18, 17, -18, -50, 64, 115, -90, -24, 75, -34, -37, 106,
      34, -88, -66, 19, -106, 55, -10, -6, -23, 64, -128, -100, -123, 106, -85, -6, -80, -71, 84, -5, 67, 30, -44, 94, -78, 15, -10, 93, -34, -84, 6, 107,
      88, -76, 1, 70, -35, 45, -101, 100, 73, 69, -32, 47, 5, 0, -124, 22, -98, 56, 5, -67, 6, 47, 30, 67, -59, 50, 61, -31, -61, 7, 24, 62,
      47, -16, -38, -105, 110, 71, 96, 101, -123, -62, -20, 31, -33, 85, -58, 100, -46, -19, 75, -56, 81, -125, 41, -58, 82, -76, -64, 100, -45, 71, 79, -27,
      29, 85, 24, 42, -34, 49, -48, 97, -7, -70, -113, -106, -69, 27, 32, 58, 41, 109, -40, 86, -104, 107, -86, -1, 124, -119, -41, -97, -90, -123, -43, -127,
      87, -70, 112, -123, 108, 64, 0, -44, 49, -81, 56, -1, -123, -42, 96, -127, 5, 56, 85, -59, 86, -41, -118, -51, 49, 118, 61, 90, 54, 13, 37, 110,
      -119, 99, -53, 23, -15, 112, -13, -114, -9, 20, 74, -75, -115, -122, -127, -17, -125, 19, -2, -76, -59, -9, 61, -5, -22, 76, 78, 125, -36, 4, -14, 62,
      51, 106, -92, 46, 64, 24, 43, 99, 67, 120, -32, -95, -38, 21, -106, 80, -59, -80, 119, 87, 4, 82, -12, -57, -121, -98, 58, -81, -28, 0, -56, 21,
      35, -100, -39, 91, 110, -69, -99, 23, 84, -14, -1, 106, 33, -61, -25, -72, 50, -104, -86, 21, -76, -36, -19, -12, -65, 54, -79, -44, 107, 28, -47, 65,
      -21, 118, 118, 21, -41, -19, -21, -87, -101, -127, 102, -57, 4, 26, 102, -2, -85, -63, 36, 16, 50, -2, 14, -3, 16, 3, 14, 35, -48, 19, 62, 120,
      -80, -57, 54, -31, 24, -50, 109, -70, 68, 59, -72, -85, -20, -44, -43, 113, -125, -124, 92, 91, -121, -106, -68, -117, -27, -45, -95, -22, 87, -119, -70, -27,
      93, -109, 115, -14, -72, -17, -63, 21, -17, 26, 99, 44, -75, -33, -60, 87, 44, 20, 111, 97, -23, -3, -101, 125, 29, -73, -85, -71, -33, 21, 84, 125,
      31, 82, -92, -22, 40, 92, -88, -103, 13, 15, 27, -100, 119, 61, -46, 43, -44, -115, -66, 44, -30, 34, 41, -109, 83, 87, -14, -128, 49, 58, -13, -11,
      -118, -73, 66, 41, 80, -41, -125, -98, -41, 73, -6, -2, -36, 18, -5, -20, 65, 30, 8, 5, 63, -10, -88, 17, 90, 71, -77, -101, 8, -108, 18, -55,
      -9, 61, -122, -53, -35, 98, -30, -29, -123, 115, 101, 103, -124, 106, -95, 52, -75, -44, -26, 47, -2, -112, 63, -35, 70, 43, -51, 74, 93, -36, 10, 83,
      58, 86, 67, 18, -51, -114, -124, -60, -6, 72, 5, -113, 14, 42, 28, 36, -11, -127, 98, 68, -53, 121, -79, 44, 1, -99, -68, -119, -107, 9, -42, 9,
      -127, -86, -110, -6, 30, -27, 66, 47, -90, -30, 35, -68, 96, 85, -44, 85, 9, -51, -2, 18, -91, -117, -127, 25, -19, -90, 70, 59, 119, -128, -115, 0,
      -36, -114, 21, 75, -87, -61, 55, 91, -27, 126, -59, 57, -115, -43, 87, -27, -28, 84, 34, -94, 81, -122, 83, 71, 92, -17, -64, -84, 93, 106, -33, -8,
      88, -7, -46, -1, 24, 45, 76, 110, 16, -86, -20, -53, 126, -92, -71, -108, 15, -97, 57, 88, 64, 4, -78, 71, 50, 27, -42, -99, -110, 6, -28, 69,
      45, 3, -14, 85, -83, -3, -46, 87, 116, 108, 84, 93, 47, -48, 122, 71, -88, -72, -64, 120, -118, 40, 53, -32, 85, 33, 117, -7, -10, 75, -52, -18,
      55, 8, -6, 52, 82, 8, 92, -127, 93, 20, 47, 100, -64, -99, -26, -25, 109, 85, -75, 73, -80, -86, -86, 5, 32, 122, -34, -106, 8, 35, -106, -19,
      -112, -94, -41, -14, 47, 42, -30, -17, 64, -10, 49, 124, 121, 102, -26, 67, -7, 16, -99, -48, 7, -32, -15, -90, 100, 55, 15, -57, -49, 100, 126, -6,
      -44, 41, 3, -24, 62, 120, -110, 90, 108, -114, 118, 18, 30, -92, 80, 68, -60, 17, 81, -54, -65, 118, -88, 55, 87, -105, -49, -89, 103, 26, 52, 29,
      -48, -76, -45, 109, -109, 83, -51, -84, 110, -23, 59, 112, 94, 24, -9, 102, -47, -7, 89, -45, 116, 50, -84, -120, -112, 101, 36, -46, -13, 52, -37, 79,
      34, -65, -96, -67, -31, -80, -45, -59, -67, -25, 118, 17, -27, -48, -62, 55, -9, -123, 52, 55, -14, 4, 5, -107, -62, 119, 91, 3, -49, 123, 112, -77,
      -64, 118, -13, 106, 22, -90, 13, -7, -11, -68, 93, 73, 84, 108, -91, -121, 125, 106, 107, 73, 111, -33, 112, -43, 50, -28, -28, 76, 125, 74, -36, 26,
      -69, 69, 5, -58, -95, -125, 36, -30, 91, -13, -41, 26, 107, 51, -63, -5, -83, 110, -37, 37, 20, -74, 38, -77, 105, 9, -8, -11, 113, 45, -73, 61,
      -10, 0, 124, 82, 72, -17, -78, -55, -11, -71, 114, 45, 22, 40, -25, 50, -67, 50, -96, -83, 24, -126, 69, 32, -14, 52, -40, -16, -102, 69, -91, 5,
      -52, 102, 122, -27, 52, 68, 27, -75, -2, -72, 108, -64, 80, 107, -99, -107, -99, -68, 27, 92, 47, -32, 112, 48, -120, -24, -78, -29, -54, -69, 74, 38,
      51, 37, -117, 120, -58, 66, -24, -93, 123, -48, 0, 99, -70, -11, 10, -76, 105, -71, 72, -4, 115, -36, -73, 71, 68, -89, 48, -83, 22, 120, -125, 29,
      40, -63, 116, -50, 24, 79, -57, -96, 45, 40, 66, 110, -124, -4, -106, 69, 104, 63, 44, 22, -80, 89, -121, 78, 47, -49, -95, -26, 4, 42, 67, 8,
      -124, 22, -108, -98, -51, 122, -74, 37, -124, 2, 25, -106, 37, 35, 47, 46, 18, -45, 4, 33, 64, 9, -72, 80, -101, -37, 63, -21, 59, 121, -34, -27,
      120, 99, 104, 122, -114, -19, -47, 53, 32, 73, -31, 100, -6, 86, 105, -111, 80, 10, -103, -84, -88, 47, -119, -126, 21, 23, -111, -71, 62, -12, 51, 23};

  // ========bias : 3x32 ============
  std::vector<float> bias = {
      -0.0112305f, 0.00976562f, -0.0415039f, -0.00146484f, 0.0605469f, -0.0625f, -0.0522461f, 0.0385742f, 0.015625f, -0.0273438f, -0.0561523f, 0.0263672f, 0.0180664f, -0.0532227f, -0.0126953f, -0.0375977f, -0.0292969f, -0.0385742f, 0.0541992f, 0.000976562f, 0.0180664f, -0.0273438f, -0.0458984f, 0.0478516f, 0.0361328f, 0.0136719f, 0.0361328f, -0.0249023f, -0.00830078f, 0.0419922f, -0.00683594f, 0.00292969f,
      0.0507812f, -0.015625f, 0.0205078f, 0.0239258f, 0.00292969f, 0.0239258f, -0.0410156f, 0.0131836f, 0.0375977f, 0.0576172f, -0.0610352f, 0.046875f, -0.0131836f, 0.0541992f, 0.015625f, -0.0395508f, -0.0185547f, -0.03125f, -0.0292969f, -0.0576172f, 0.0258789f, -0.0351562f, -0.00244141f, -0.00292969f, -0.059082f, -0.0449219f, -0.0141602f, -0.0273438f, 0.0454102f, -0.0493164f, 0.0234375f, 0.0600586f,
      -0.0336914f, 0.0595703f, -0.0473633f, -0.0576172f, 0.0551758f, 0.0576172f, 0.0322266f, -0.0385742f, -0.0517578f, 0.0283203f, -0.0180664f, 0.0551758f, -0.0507812f, -0.0195312f, -0.0566406f, 0.0410156f, 0.0498047f, 0.0151367f, -0.0615234f, 0.0380859f, -0.0498047f, -0.0488281f, 0.0253906f, 0.00976562f, 0.0444336f, 0.0576172f, 0.00585938f, 0.0200195f, -0.012207f, 0.0292969f, 0.00244141f, -0.0131836f};

  //========attention_mask : 1x128 ============
  std::vector<int32_t> attention_mask_fp32 = {
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0};
  std::vector<MLFloat16> attention_mask(attention_mask_fp32.size());
  for (size_t i = 0; i < attention_mask_fp32.size(); i++) {
    attention_mask[i] = MLFloat16(1.0f * attention_mask_fp32[i]);
  };

  // ========global_weightq : 96x32 ============
  std::vector<int8_t> global_weightq = {
      -100, 58, -115, -104, 104, 120, 81, 15, -55, -7, 106, -37, 18, -90, 113, -69, -44, -115, 71, 2, -11, -83, 43, -7, -90, -22, -8, 107, -8, 43, 45, 110,
      80, 33, 8, -92, -60, -67, 125, 11, 17, -6, 27, -91, 64, 83, -124, 20, -11, 11, -18, -109, 48, -6, -57, -104, -19, 61, 103, 58, 96, -97, 21, 115,
      -72, -86, 126, -67, -128, 77, -48, 25, -71, -68, -65, 13, -1, -38, -111, 21, 11, 93, -48, 90, -62, 22, -94, 67, -24, 103, -66, -14, -107, -60, -123, 93,
      114, 29, -114, -119, -8, 94, 12, 58, 103, 3, -103, 45, -55, -81, -45, -60, 39, 9, -110, -27, -45, 70, 0, 89, -114, 83, -98, 31, 41, -78, 51, 10,
      54, 4, 9, 58, 14, 61, 52, 91, -116, 109, 9, -67, -110, -26, 115, 71, -59, -64, -24, -77, 52, -80, 19, 17, 22, -29, -71, -90, -55, -16, 115, -24,
      -8, 44, 46, 74, 93, 105, -91, -23, 17, -39, -42, -102, -84, -1, -102, 47, 21, 31, -123, 65, 41, 70, -70, 16, -110, -50, -60, -58, 116, -31, -98, -115,
      70, 69, 74, 70, 21, -80, -38, -94, -2, 77, 105, -11, -71, -22, -80, 65, 94, 12, 47, 108, 59, 5, 27, 103, 102, 125, -109, -104, 118, 57, -112, -11,
      -122, -68, 27, 22, 116, -52, 91, -44, -54, -61, -73, -59, 50, 74, -47, -90, 125, -2, -103, 41, -56, -128, -97, 22, 63, 97, 66, -4, -34, 62, 3, -124,
      -120, 115, 3, 49, -32, 103, 0, -40, 92, 88, 4, 58, 102, -91, -83, -33, -69, 22, -78, 34, 20, -114, 126, -46, -77, -16, -83, -116, -74, 2, 118, -58,
      87, -8, 26, 103, -20, -28, -39, 110, -5, 114, 47, -108, -34, -120, -58, 48, 75, 57, -46, -127, 110, -1, -121, 113, 37, -119, 97, 39, 45, 114, -62, -124,
      -69, 70, -4, 25, -39, 60, 61, -67, -48, 97, 99, 38, 49, 115, -24, -93, -67, 84, -25, -59, -80, -39, 57, -42, 80, 25, 23, 118, 69, 34, -27, 34,
      -19, 82, 114, -26, 76, -39, -43, 36, -61, 76, 39, 124, -85, -80, -114, -26, -111, 29, -95, -56, -3, 21, 39, 22, 91, -56, 93, 103, 21, 108, -64, -9,
      -62, 24, 48, -66, 13, 81, 10, -56, -114, -113, 12, -116, -112, 98, -98, -38, -115, 8, 101, 6, -64, 75, 41, 48, 126, -52, -16, -93, 25, 77, 108, 8,
      -68, -84, -64, 1, -123, -30, -77, -76, 57, -92, -98, 54, -93, 71, -107, 52, 38, -9, 93, -88, 93, -58, 86, -74, 71, -40, 65, 85, 91, 53, -94, -75,
      122, 61, -118, -86, -100, 104, -108, 41, -14, -104, 19, 8, 79, -60, -24, -119, -114, -30, -37, 108, 44, -47, -128, -54, -50, -39, 48, 23, 47, -36, -126, 81,
      1, 97, 34, -14, -88, -89, -45, 18, 86, -11, -73, -21, 44, -78, -64, -18, 100, 51, -103, -3, 107, -25, 105, 66, -76, 86, 65, 105, 32, -20, 58, -106,
      -55, -91, 60, -6, -21, -44, 121, -52, 48, -16, 12, -7, -39, -124, -119, -6, -29, -72, -12, -100, 101, 22, 87, -53, 86, -87, 100, 4, -98, -5, 15, 21,
      -114, -69, -6, -13, 106, 49, -83, -21, 91, -53, -108, -52, 37, -63, -83, 8, 53, 69, -39, -42, -53, 58, -89, 23, 98, 100, 50, -38, -81, 45, -2, -105,
      110, 32, 69, -127, -106, 33, -100, -64, -28, -1, -10, -96, -25, 72, 27, 106, 106, 45, -77, 105, 121, 32, 31, 14, -45, 27, 21, 112, -12, -82, 59, -16,
      76, -84, 68, -29, -97, -43, 111, 7, -43, 64, 102, -87, 7, 62, 121, 46, -8, -39, -64, -61, -12, -120, 48, 79, 75, 65, 68, -93, 116, -128, 76, 115,
      34, -73, -67, 77, -10, 66, 120, -7, 34, -70, 122, -32, -71, 73, -114, 39, -65, 98, 61, 37, 28, -46, 17, 116, 41, -32, 62, -72, 26, 83, -23, -30,
      65, 18, -65, 82, 84, 41, -106, 19, -73, 36, 5, 38, -122, 1, -115, -45, 119, -42, -93, 120, 15, -54, 114, -37, -61, 49, 27, 29, 109, 117, -2, 78,
      -13, -75, -70, 115, -64, 61, -117, 75, -127, 8, 32, 35, -73, 94, -124, -39, -34, -68, -80, 80, -106, 31, -95, -1, -44, 10, 30, -57, -4, -31, -69, -54,
      90, -17, -127, 55, 101, 63, -38, -27, 22, -10, 90, -106, 12, 39, 20, 54, 19, 38, 46, 5, 102, -73, -35, 28, -97, 52, 23, 29, 35, -107, -93, 92,
      -78, -102, 96, -51, 106, 17, 88, 108, -58, -37, -19, -81, -34, 87, -29, 38, 1, 94, 99, -87, -92, 16, 113, 75, 5, -92, 22, -58, 36, -80, 6, 49,
      71, -51, 42, -73, 71, 65, -49, -39, 24, -49, -91, 41, 41, -65, 93, 74, 88, 17, -55, -89, 43, -15, -110, -74, -92, -120, 53, -96, 82, 92, -96, -60,
      -44, -98, 75, -114, -13, -110, -103, -23, -128, -78, -44, 27, -76, 7, -91, 11, -85, 69, -66, -8, -57, -81, -26, -117, 19, 60, -12, -123, -66, -55, -24, -42,
      95, -76, 24, -51, -102, 78, 33, 19, 32, 88, -88, 94, 11, -59, 31, -118, 109, 67, -78, -90, 104, -24, 5, 119, -15, 121, 100, -3, 23, 42, -32, -36,
      -118, -80, 114, -114, 63, 24, 35, -50, -97, 11, -30, 39, 77, 39, -71, -96, 9, -124, -73, 45, 19, 93, -68, -1, -24, -123, -15, -24, -117, -73, -11, -124,
      -80, -32, 85, 33, 65, -96, -37, -96, 63, 77, -123, -117, 13, 81, 48, 12, 26, 51, 39, -124, 1, -72, -22, 66, -4, 115, -48, 118, 21, 55, 53, 19,
      -121, -51, -38, -46, -6, 69, 99, 43, -42, -21, -46, -86, 7, 72, -6, 60, -79, 78, -4, 97, -37, 34, -104, 66, -54, -39, -80, 69, 115, 124, -2, -95,
      -31, -97, 57, 60, -75, -75, -44, -126, 14, -69, 4, -121, -27, -118, 106, -25, 41, -18, -2, -58, -87, -103, 61, 64, -61, -43, 44, 105, -65, 104, 98, -103,
      -20, -121, -33, 124, -77, -107, 44, 7, -128, 79, -76, -38, -37, 120, 91, -81, 80, -111, -9, 39, 83, -38, 80, -98, 87, 18, 25, -122, -39, 115, -39, -100,
      -82, 100, -116, -121, 18, -87, -11, -28, -53, -15, 28, -78, 121, -29, -69, 13, 2, -24, 104, -36, -91, -11, 63, -114, 6, -74, 78, -106, 83, -107, -99, -54,
      73, 36, -128, -112, -2, -20, 64, 74, -56, 41, 16, -40, 53, 87, 124, 56, 115, 60, -4, -74, 93, 78, -28, 47, 95, 14, 2, 11, -100, -74, -8, -30,
      -99, 28, -93, 87, 76, -61, -69, 104, -20, -14, 47, -105, -64, 49, -49, -101, 64, 3, -23, -103, 100, 66, 126, -127, 52, 28, -82, 59, 1, 103, 107, -107,
      -45, 98, -83, 90, 104, 126, 80, -93, 59, 114, 71, 7, 72, -101, 80, -16, 122, 61, -27, -57, -57, -49, -96, -51, -69, -116, -83, 1, -55, -124, -122, 60,
      32, -116, -75, -76, -49, 65, -39, -99, -38, 39, -81, -51, -58, 21, 125, -115, -87, -77, 36, 2, -37, 75, 109, 68, -68, -111, 69, -60, 115, -24, 105, -72,
      -17, -117, 122, -109, 63, -60, 72, 122, -86, 119, -97, 43, 112, -50, 53, 34, 25, -93, -13, 72, 88, -43, -126, 58, -123, 85, 103, -100, -34, 94, 23, -46,
      122, 40, -108, -122, -69, 55, -26, 25, 60, -46, -114, 26, -109, -92, -56, 48, 76, 108, -125, -48, -38, -126, -22, -29, -71, 43, -66, -75, 54, 19, -7, -90,
      -91, 23, -78, -48, -111, 1, 30, 12, -46, -13, -68, 53, 8, -33, -86, 117, -64, -113, 99, 94, 86, 55, -40, -122, 118, -72, 107, 75, -95, -41, 119, 94,
      -77, 64, -116, -21, -122, 72, 85, 17, 82, -32, -43, -17, -35, -114, -89, 71, 78, -114, 91, -41, -46, -78, 66, 27, -44, -25, -124, -37, 106, -58, 42, -46,
      -3, 72, -39, 60, -111, 43, 62, 56, -9, -55, 67, 13, 99, -116, -48, 39, -15, 92, -70, 68, -82, -54, 67, -124, 106, 46, -119, 48, 45, -89, -73, -79,
      10, -51, -104, 88, -38, 6, -100, -118, -2, -107, 57, 106, 120, -128, 112, 37, -73, 21, -18, 17, 14, -3, 96, 28, 2, 36, 77, -54, 53, -36, -50, 90,
      12, -61, 16, -48, 84, -120, -54, 17, -30, -60, -57, 21, -31, -108, 81, 33, 9, 36, 57, -99, 37, 15, -60, -36, 93, 119, -65, 86, -115, -72, 102, -128,
      26, -26, -106, -85, -124, 114, -103, -63, 71, 17, -101, 50, -74, -58, -20, -53, 80, -52, 17, -120, 68, 109, 24, 21, 96, 98, -34, 36, 101, -21, -44, 122,
      52, 20, -125, -29, 118, 94, 74, 34, -23, -3, 69, 71, -114, 79, -105, -80, 30, -13, -87, -105, 12, -24, -77, 42, -48, 106, 78, 113, 42, -105, 115, 83,
      -79, 49, -3, 88, -126, 106, -104, -66, 1, 124, 15, 52, 60, 116, 69, -127, -68, -112, -5, 61, -40, -65, 4, -126, 60, 112, 72, 80, 83, 42, 24, 115,
      -54, -78, 11, 69, -114, -34, 34, 112, -59, 68, -66, 86, 94, -115, 30, 62, -22, 5, 88, 87, 93, 72, -33, 44, 31, -33, -88, -77, -76, -51, 107, -115,
      -39, -106, -94, -121, 43, -30, 36, 86, 52, 24, 113, -46, 68, -86, 3, -103, 53, -78, -88, 39, 4, -45, -41, 60, 118, -85, -19, -80, 73, 81, 101, 124,
      -86, 34, 80, 44, -29, -82, 4, -122, 0, -49, 101, -68, 108, -96, 44, 19, -5, -18, -46, -44, -91, -29, -40, 41, 45, -30, 119, -58, 103, 10, 68, 63,
      78, -35, -69, -11, -55, 21, -80, -46, 10, -20, -6, -19, -90, -8, 33, -83, -45, -101, -68, -128, -113, 55, 77, -115, -120, -107, 98, -58, 102, 67, 102, -47,
      -42, 62, -32, 115, -86, 119, -42, -101, 84, 82, -37, -62, -122, -32, -25, 126, 1, 65, -73, -107, -94, 72, -31, -78, 74, -23, 22, -84, -9, 60, 36, 46,
      80, 54, 11, 16, 5, -106, -95, 32, 87, -111, 9, 14, 65, -46, 124, 112, 116, 28, 96, 12, -28, 107, 23, -67, -41, 37, 54, -61, 36, -73, 117, 31,
      11, -3, -100, 53, -53, -17, -68, -5, 5, -106, -82, 99, -74, -107, 67, -39, -21, -110, 32, -47, -122, 15, -89, -73, 100, -86, 101, -69, 12, -74, 11, -124,
      58, -86, 54, -75, -95, 20, 52, 120, 83, 107, 102, -23, 123, -1, 24, 24, -92, -15, -10, 21, 124, 117, -51, -78, -101, 53, 108, -69, -91, -107, 28, -106,
      -37, -114, -107, 14, 89, -75, -25, -75, -20, 105, -101, -51, -119, 37, 64, -108, -6, -82, 71, -105, 103, 18, 26, -54, -3, -77, 31, 120, 50, -66, 103, -40,
      -103, -11, -68, -93, 6, -14, -121, 80, -122, -128, 101, -45, 89, 56, -50, 79, -11, -97, -106, 26, 1, -124, 76, -28, -88, -67, 16, 114, 88, 88, 115, 3,
      -119, -109, 13, -103, -46, -102, -119, 108, 32, 59, -103, 76, -47, -67, 99, 14, -59, -37, 2, 9, 124, -63, 106, -89, 94, 122, -125, -35, -113, -19, -23, 39,
      90, 121, -80, 123, 29, -98, 23, -79, -84, -65, -125, 65, 19, -119, -79, -54, -76, -13, 70, 17, 102, -17, -93, 99, -31, 111, -19, 105, -123, 14, 46, 113,
      47, 15, 109, 7, -99, -63, 21, -118, -8, -120, 101, -14, -95, 55, 87, -33, -95, 62, -70, 95, -106, 28, -20, -64, 74, 15, 58, -128, 104, 78, -104, -92,
      81, 83, -26, -77, 28, -87, 78, -75, -2, 95, 109, 92, -1, -4, 123, -83, -113, 86, 40, 56, 18, 85, 21, 123, -102, -65, -101, 92, 112, -125, -30, -82,
      -93, -104, -29, -91, 94, 46, -110, 108, -63, -85, -66, 116, 106, 107, 91, 9, 83, 53, -87, 99, -91, 81, -84, -23, -107, -70, -7, -116, -92, -43, -70, -13,
      3, -65, -98, 7, 64, 96, -41, -22, 76, 78, 34, 13, 54, -119, -88, -85, -46, -110, -38, 57, 42, 86, -88, 11, -82, -97, -36, -14, -4, 73, -121, -26,
      -35, -58, -38, 59, -101, 125, -125, 45, -58, 40, 51, -114, -122, -2, -90, -109, -67, 29, -12, -92, 42, -4, 21, 82, -125, 91, -33, 80, -43, 76, 1, 106,
      56, 37, 37, -33, 33, 16, -52, 107, -88, -35, 27, 94, -113, 37, -72, 97, 22, 18, 117, -11, -104, -72, 30, -37, -75, 60, 89, -47, 93, 50, -104, -87,
      66, -16, 29, -9, -80, 115, -106, 23, 91, -100, 56, -98, 105, -34, 119, 45, -123, 69, -105, 93, 18, 15, -49, -51, 107, -39, -63, -47, 38, -46, 76, 45,
      -43, 33, -51, -62, 19, -35, -71, 75, 84, -11, -55, -99, -32, 62, 31, -28, -67, 55, -124, -7, -35, -2, 49, 57, -109, -86, 10, 50, 65, -14, 75, 6,
      -106, -14, 12, 70, -72, -14, 95, 112, 54, 47, 19, 6, -50, -20, -70, -83, -85, 83, 14, -83, 35, -37, -62, -28, 13, 114, 61, 96, -58, -59, 88, -75,
      59, 62, -10, 94, 3, 69, -7, 45, -22, -45, 117, 19, 125, 37, -55, 39, 110, 13, 112, -119, -19, -29, -79, -54, -31, 7, -6, 85, 75, 21, -120, 97,
      102, 83, 7, 90, -74, -70, -71, -126, 3, -18, 68, -7, 96, 87, -103, 20, -38, 83, 95, -118, -48, -9, -4, 19, -31, -78, -123, 78, -37, -3, -126, 76,
      40, 84, -58, -23, -93, 1, 68, 69, -65, -4, 30, 13, 67, -49, -60, 3, -67, -37, -56, -60, -110, 88, 24, -125, -95, 105, -86, -41, -115, 1, 31, 38,
      45, 75, -82, 95, 79, -2, 108, -33, 89, 90, 13, 107, -110, -85, 125, 75, 88, -17, 86, -90, 16, -124, -37, -77, -60, 9, 28, -69, -118, 99, -17, 87,
      16, -26, -49, 53, -56, 64, 18, 99, 30, -42, 46, -28, -84, -61, -64, 122, 96, 4, 82, 107, 112, -47, -124, -117, 19, -54, 99, -38, -91, -58, -116, 123,
      -91, -56, 67, -50, 106, 24, -92, -15, -66, 98, -68, -4, 98, 118, 74, 12, 29, -46, 86, -106, -33, 81, -6, 79, -89, 97, 32, 93, -36, 16, -66, -32,
      -62, 114, -59, 41, 70, 36, -105, 99, 61, -63, 14, -74, 83, -56, -101, -36, -85, -19, -123, -90, -107, -42, -95, -83, -38, 52, 3, 109, -115, -41, -109, 37,
      33, 48, -116, -110, -17, -111, -124, 44, 97, -97, -36, 65, -92, 119, 49, -36, -50, 101, -66, -101, -42, -83, -21, 65, 102, -125, 60, -87, 80, 70, -66, -72,
      104, -108, 60, -26, -37, -89, 69, 24, 12, -17, -7, 77, 52, -88, 32, -71, 104, 25, -2, -26, 65, -28, 31, -92, 7, -35, -65, -3, 68, 102, -111, 75,
      -56, 88, -90, 104, 103, -100, -35, 107, 99, -3, 93, 100, 125, 17, 62, 105, -27, -90, 26, -34, 20, 67, 22, 30, -105, -90, 112, -47, 22, 56, -39, -28,
      122, 102, -102, 98, 56, 114, 120, 21, 2, 12, -124, 59, -40, 0, -86, 116, 68, -12, 17, -6, -48, 42, -87, -27, 102, 49, 85, -100, 49, -120, 19, -108,
      -3, 95, -71, 122, 51, -33, 113, -109, -13, -11, 53, -111, -6, -120, 102, 84, -21, 29, 22, 46, -128, -24, 86, 80, 98, 33, 4, 90, 75, 89, -28, -101,
      107, -81, -25, -66, 98, 44, 106, -58, 45, -47, -54, -111, -34, 26, -47, 6, 58, 4, 70, 11, 36, 13, 110, -107, -65, 85, -71, 105, -89, -22, 15, -15,
      34, 2, -117, -81, 123, 94, 70, -108, -66, 125, -55, 9, -67, 93, -111, 122, -36, -110, -89, -29, 115, -8, -104, 9, 84, 118, -68, -92, 77, -35, 23, -124,
      99, -50, -79, -103, 120, -45, 107, -10, -12, 113, 51, 32, -67, -68, 30, -15, 64, -42, 70, 83, -75, 114, 95, -66, -19, 5, 43, 114, -117, 104, 37, 112,
      118, 102, -11, -116, 114, -66, -48, 41, -68, 66, -3, 109, 21, 9, 69, 98, 114, 110, 24, -118, 77, 105, 120, 89, -51, -40, 97, -72, 58, -81, 67, -115,
      -18, -33, -77, -100, 106, 106, 92, -122, 74, -55, 112, 75, 79, 107, -7, 39, -111, 27, -22, 0, 55, 100, -114, -99, -37, -115, -58, 81, -86, -121, 90, 87,
      -27, 40, -127, 95, 36, -79, -113, -44, 73, -37, 42, -37, -90, 60, -15, -40, -68, 92, -51, -12, -110, -20, -117, 82, 91, -126, 112, 61, -118, -13, -59, 118,
      13, 9, 12, -122, 42, 121, -70, 104, 82, -119, -23, -57, 36, -69, 70, -101, -69, -76, -98, -54, -17, -38, 23, -23, -24, -115, -62, -9, -55, -87, 102, -4,
      -52, 123, -11, -114, -28, -122, -21, -100, -110, 17, 98, -108, 123, -76, -104, 42, 52, 82, 40, 6, -110, 71, -126, -27, -4, 121, -119, 24, 31, -24, -61, -85,
      -72, 0, -74, -21, -78, -110, -10, 6, -6, -25, -49, 105, 84, 115, -95, 35, -87, 11, -84, -49, 113, -38, -43, -29, -42, -29, -17, -79, 16, -125, 105, -95,
      108, -38, 42, 88, -95, -101, -12, -121, 76, 113, -51, -4, 16, -107, -5, -6, 115, -5, 45, 70, 83, -43, -17, 106, -50, -59, 84, -6, 45, 13, -86, 25,
      -69, 119, 8, 68, 77, 53, 122, 39, -127, 2, 40, 21, -126, -17, 93, 85, 69, -24, -41, 78, 45, 114, 71, -104, -46, -37, 80, 96, -91, 72, 41, -109,
      14, 110, 31, 45, 111, 113, -28, -103, 26, -96, -30, -61, -84, -11, -87, 73, 30, 24, 89, -115, 31, 17, 74, -40, -90, -111, 64, 117, -114, 123, 53, 8,
      -25, -123, -20, 58, -74, 1, -23, -58, -21, -23, 106, -73, -99, -120, -24, 105, 125, 7, -34, -75, 103, 35, -83, 43, -91, 4, 30, 86, -78, -108, 74, -89,
      125, 23, -17, -28, 93, 22, -123, 96, -85, -42, 121, 63, -50, -47, 17, 45, 122, 46, -108, -18, -112, -76, -73, -62, 54, -93, -53, -100, 99, -90, -28, -17,
      -115, 100, -49, 68, -7, -49, 48, -84, 80, 59, -9, -24, -63, -104, -88, -80, 18, -92, 87, -93, 1, -115, 80, -77, 10, 74, 53, -52, -82, 19, -125, 118};

  // ========global_bias : 3x32 ============
  std::vector<float> global_bias = {
      0.0078125f, -0.0605469f, -0.00341797f, 0.03125f, 0.00927734f, 0.0229492f, -0.0449219f, 0.0449219f, 0.0419922f, -0.0605469f, 0.0483398f, 0.000488281f, 0.0478516f, 0.0415039f, -0.00244141f, 0.000488281f, -0.00439453f, -0.0400391f, -0.0522461f, 0.0517578f, -0.0332031f, 0.0454102f, -0.0512695f, -0.0102539f, -0.0288086f, -0.0200195f, -0.0576172f, 0.0566406f, -0.0205078f, 0.0566406f, 0.0605469f, 0.000976562f,
      -0.0146484f, -0.0151367f, -0.015625f, -0.0517578f, 0.0429688f, -0.0571289f, 0.00390625f, 0.00878906f, 0.0351562f, -0.0219727f, 0.0385742f, 0.0444336f, 0.00927734f, -0.0102539f, 0.0219727f, -0.0390625f, -0.0561523f, -0.0541992f, -0.0493164f, 0.00634766f, -0.019043f, 0.0229492f, 0.0615234f, 0.0551758f, 0.0126953f, 0.0170898f, 0.0478516f, 0.0126953f, 0.0205078f, 0.0288086f, 0.0537109f, 0.0f,
      -0.00634766f, 0.00390625f, 0.0424805f, 0.0161133f, -0.0600586f, -0.00195312f, -0.0454102f, -0.0615234f, 0.0268555f, 0.0146484f, -0.0551758f, -0.0600586f, 0.0576172f, 0.0f, 0.0405273f, 0.0185547f, 0.015625f, 0.0463867f, 0.0410156f, -0.0356445f, 0.0395508f, 0.0541992f, -0.00292969f, 0.0126953f, -0.0478516f, -0.0234375f, 0.0175781f, -0.00244141f, 0.000976562f, -0.0195312f, 0.0576172f, 0.012207f};

  //========global_attention_mask : 1x128 ============
  std::vector<int32_t> global_attention_mask = {
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

  std::vector<int8_t> gold_result = {
      127, 16, 16, 115, -79, -43, -99, -21, -114, 26, -76, -128, -39, 127, -128, -128, 2, 78, -128, 127, 114, -50, -40, -36, 92, -36, -62, -128, -128, -128, -62, 52,
      -40, 20, -114, 127, 80, 54, -102, 24, -78, -70, 10, 127, -126, -60, 78, -6, 118, 110, -10, -28, -78, -128, -116, -30, 66, 48, 114, -128, -16, -6, -128, 127,
      -128, -108, 62, -128, -66, 72, 12, -34, 0, 127, -128, -128, -32, 127, -36, -56, -128, 42, -114, -60, -16, -128, 127, 20, -128, -64, 46, -128, 6, 68, 127, 16,
      127, -102, 80, 86, -2, -128, -112, -124, -4, -36, -66, -2, -48, -114, -128, -102, 80, -128, -82, 86, 92, -60, 127, -72, -128, -24, 94, -98, -128, -108, 114, 32,
      -90, -84, 12, 64, 66, -56, 4, -66, -120, 18, -44, 127, 127, -128, 127, 127, 24, -72, 14, 116, 50, -116, -86, -102, -20, 48, 112, 127, 60, 50, -128, 48,
      -88, -38, 6, -128, 26, 86, 56, 127, 98, 44, -76, 114, 20, 127, -128, 124, 127, 127, 127, 127, 127, 127, -128, 50, 62, -46, -64, 127, -36, -104, -128, 34,
      -40, 64, 127, 40, -12, 70, 114, -72, 42, 127, -128, -62, 127, -14, -54, 72, 127, -28, -128, 127, 116, 127, -24, -128, 127, -68, 90, 12, -8, -36, 127, -30,
      -104, -18, 127, -128, -128, 127, 127, -12, 127, 127, -76, 60, 127, 127, 127, -16, -66, 102, 56, 82, -128, 127, 96, -36, -58, -10, -128, 127, -48, 94, 12, -128,
      127, 16, -44, -90, 127, 112, 104, 38, 80, 104, -128, 127, 127, 4, -8, 44, 98, -8, 127, 80, -108, 54, 16, 26, 127, -74, 40, -24, -128, 102, -92, 0,
      -128, 36, -36, -88, -128, 127, 127, 127, 127, -62, -90, -14, 14, 127, 127, -50, -82, 64, -10, -6, -128, -120, -128, 18, -128, -30, -20, 102, -80, -28, -16, -124,
      -128, 36, -36, -88, -128, 127, 127, 127, 127, -62, -90, -14, 14, 127, 127, -50, -116, -30, -128, 4, -70, -128, 127, -26, -128, 28, -92, 28, 127, -66, 20, 56,
      127, 22, 76, 16, 127, 127, 110, 68, -128, -46, -64, -40, -58, -118, -106, 127, -128, -128, -80, 56, -128, -70, 127, 32, -128, -66, 48, 86, 18, -118, 88, 22,
      -82, 118, -128, -86, -8, 54, 127, 68, -128, -128, 127, -42, -72, -20, 22, -96, 4, 127, 30, -24, -128, 106, -74, -16, -50, -112, 2, -128, -22, -90, -104, -42,
      96, -44, 104, 72, -76, -36, 112, 68, -128, -128, 38, 114, -4, 40, -10, -128, -108, -98, 127, -128, 84, -128, -74, 50, 127, 127, 34, -128, -128, 74, 127, 84,
      -82, 98, -128, -110, 92, 62, -98, 44, -28, 106, 90, -18, -128, 60, -30, 6, 24, -72, 14, 116, 50, -116, -86, -102, -20, 48, 112, 127, 60, 50, -128, 48,
      110, -22, 68, 62, 22, 74, 84, 48, 22, -128, 80, 108, 50, -100, -22, 80, 80, -128, -82, 86, 92, -60, 127, -72, -128, -24, 94, -98, -128, -108, 114, 32,
      110, 34, 18, 127, -128, -40, -12, -46, -82, -80, -24, -128, -76, -128, -60, 127, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      -51, -128, 36, -61, 59, 12, -25, -22, 76, -28, 50, 127, 124, 46, 31, -1, -75, -90, 127, 127, 103, -128, -27, 8, 22, 6, 127, -44, 12, -123, -118, 36,
      70, 118, -54, 24, -50, -112, -4, -84, 14, -128, 127, 30, 82, 30, 98, -124, 116, -72, -128, -50, -74, -104, -128, -77, 2, -121, 87, 72, 49, 127, 115, -46,
      52, 20, 98, -94, -74, -56, 82, 56, -68, 127, 127, -46, -66, 124, -8, 114, -26, 127, -4, -128, -38, -128, -126, 78, -58, -30, -128, 72, 4, -128, 127, -128,
      118, 106, -128, 14, 96, 108, 127, 127, -120, -24, 32, 127, -12, 52, -22, -46, -8, 16, 127, 2, 112, -128, -36, 73, 88, 92, -70, -24, -128, 112, -114, 17,
      -98, -128, 6, 127, -128, 52, 48, 48, 54, 127, -128, 127, 42, 18, 82, 127, -128, -68, -128, -50, 32, 34, -92, -128, 56, 72, -52, 127, -128, -2, -108, -64,
      60, -84, 10, 127, -18, -68, 127, -24, 28, 127, -48, 20, -104, 20, 86, -86, -84, -34, 0, 18, 38, -128, -48, 106, 54, 127, 122, -16, 46, -128, 42, 78,
      110, 34, 18, 127, -128, -40, -12, -46, -82, -80, -24, -128, -76, -128, -60, 127, 78, 16, 124, -128, -50, -128, -4, 86, -128, 20, -58, -58, -44, -128, 108, 127,
      127, -38, -18, -24, -66, 16, -102, -12, 74, -128, -58, 88, -30, -88, 30, -12, 52, -32, 100, 28, 92, 6, -128, -48, -100, 88, 127, -46, -128, -128, 24, -44,
      -2, 16, -6, 42, -14, -2, -76, -60, 32, -98, 76, 110, 90, 22, 68, 50, -128, -128, -26, -128, 114, -60, -44, 122, -126, -128, 127, 48, -128, 102, 112, -74,
      82, -76, -128, 112, -128, 114, 38, -104, 88, -4, -128, -20, -96, -92, 127, 127, 48, 127, 48, 94, 38, 84, 127, 68, 36, 127, -128, -58, 127, 86, -94, 72,
      82, -76, -128, 112, -128, 114, 38, -104, 88, -4, -128, -20, -96, -92, 127, 127, 44, 127, -128, -72, 66, 127, -32, -128, 127, -18, 124, 127, -98, 127, -10, 4,
      82, -76, -128, 112, -128, 114, 38, -104, 88, -4, -128, -20, -96, -92, 127, 127, 116, -74, -128, -50, -76, -104, -128, -78, 0, -124, 88, 72, 48, 127, 114, -46,
      14, -96, -36, 0, -128, 18, 126, -84, 127, 104, -128, 98, -128, -128, 94, 58, 78, 16, 124, -128, -50, -128, -4, 86, -128, 20, -58, -58, -44, -128, 108, 127,
      -14, -82, 127, -128, -128, -52, -24, -128, 106, 120, -100, 127, 52, 127, 82, 18, -116, -66, 127, -128, -128, -128, 14, 127, 48, 88, -98, -48, -68, -128, -36, 2,
      28, -128, 127, -26, 127, 127, 28, -128, 60, 20, -54, 78, 64, 127, -34, -88, -128, 127, 86, -100, -28, -2, 94, 16, -128, -102, 127, 127, -128, -128, -128, 26,
      58, 100, -88, 110, -128, 127, -84, -102, -4, -128, -32, -8, 50, 70, 28, 34, 127, -66, -16, -82, 14, 46, -42, -128, -96, -70, -26, -128, -56, 127, -128, 127,
      2, 12, -128, -72, -104, -100, 60, -30, -108, -38, -128, -20, -114, -128, -128, -70, 127, 18, -128, 40, 124, -34, 127, 127, 127, 120, 58, 52, 127, 8, 124, -52,
      -94, -126, -128, -128, 127, -128, -54, 86, 10, 127, 127, 12, 104, -44, 127, 127, 20, -28, 127, 127, 127, 127, 127, -50, 80, 12, 127, -128, 127, 127, -64, 127,
      96, -28, 66, 127, -128, 60, -76, -78, 82, -128, -50, -74, -128, -42, 58, 68, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      118, 106, -128, 14, 96, 108, 127, 127, -120, -24, 32, 127, -12, 52, -22, -46, 127, -48, 12, 30, -24, -128, 127, -128, 16, 56, -118, -74, 127, -128, -94, 127,
      -128, -36, 2, 4, -128, 56, 126, 10, -4, 127, 8, 127, -2, 110, 78, 88, -66, 22, -128, -128, 18, 10, -54, -48, -52, -128, -82, -18, -128, -98, 127, -128,
      -128, -30, 30, -128, 56, -128, -128, 90, 68, -128, 2, 10, 38, -110, -54, -128, -4, 127, 0, -44, -128, -128, 40, 127, 127, 14, 88, 127, 58, -128, -66, -128,
      -86, 38, 127, -68, 127, 108, -128, 30, 16, -128, 52, -96, 127, 122, -128, -16, 44, 127, -128, -72, 66, 127, -32, -128, 127, -18, 124, 127, -98, 127, -10, 4,
      -128, -30, 30, -128, 56, -128, -128, 90, 68, -128, 2, 10, 38, -110, -54, -128, 127, -66, -16, -82, 14, 46, -42, -128, -96, -70, -26, -128, -56, 127, -128, 127,
      -86, 38, 127, -68, 127, 108, -128, 30, 16, -128, 52, -96, 127, 122, -128, -16, -92, 50, -104, 52, 0, 10, 104, -42, -112, -112, 18, 36, 42, -38, 18, 127,
      58, 100, -88, 110, -128, 127, -84, -102, -4, -128, -32, -8, 50, 70, 28, 34, 10, 88, -104, 12, -102, -128, -128, -58, 68, -26, 122, 58, -128, -128, -96, -128,
      127, 56, -88, 127, -54, -28, -68, -56, -96, 32, 52, -60, 127, 54, 92, -80, 116, -74, -128, -50, -76, -104, -128, -78, 0, -124, 88, 72, 48, 127, 114, -46,
      -94, -126, -128, -128, 127, -128, -54, 86, 10, 127, 127, 12, 104, -44, 127, 127, -2, 78, 46, 6, -40, -128, 127, -90, 127, 127, -128, -128, 12, -32, -128, 127,
      14, -96, -36, 0, -128, 18, 126, -84, 127, 104, -128, 98, -128, -128, 94, 58, 127, -74, 127, -72, -2, 84, -52, -128, -128, -92, -88, -128, -26, 124, -40, 38,
      28, -128, 127, -26, 127, 127, 28, -128, 60, 20, -54, 78, 64, 127, -34, -88, 44, 127, -128, -72, 66, 127, -32, -128, 127, -18, 124, 127, -98, 127, -10, 4,
      28, 72, -20, -128, -128, 72, 127, 96, 32, 56, -96, -114, -100, -58, 18, 127, -8, 16, 127, 2, 112, -128, -36, 74, 88, 92, -70, -24, -128, 112, -114, 16,
      -94, -126, -128, -128, 127, -128, -54, 86, 10, 127, 127, 12, 104, -44, 127, 127, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      22, 124, -73, -128, 109, -128, -42, 127, 127, -34, 11, -4, -37, -128, 64, -128, -53, -72, -126, -128, 45, 42, -31, 72, 127, 127, 85, 104, -128, 4, -34, -106,
      127, 127, 0, 127, 1, -110, 73, 36, -115, 38, 127, -55, 122, 21, 127, -4, -8, 16, 127, 2, 112, -128, -36, 74, 88, 92, -70, -24, -128, 112, -114, 16,
      -16, -64, 46, 127, -128, -20, 26, 127, -42, 4, 127, 118, -54, 127, 38, 127, -26, 127, -4, -128, -38, -128, -126, 78, -58, -30, -128, 72, 4, -128, 127, -128,
      -94, -126, -128, -128, 127, -128, -54, 86, 10, 127, 127, 12, 104, -44, 127, 127, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      -2, 16, -6, 42, -14, -2, -76, -60, 32, -98, 76, 110, 90, 22, 68, 50, 127, -48, 12, 30, -24, -128, 127, -128, 16, 56, -118, -74, 127, -128, -94, 127,
      26, -120, 127, -128, -52, -82, -68, -64, 48, 18, -128, -80, -128, -100, -128, 127, -128, 127, 86, -100, -28, -2, 94, 16, -128, -102, 127, 127, -128, -128, -128, 26,
      127, -86, 127, -128, 48, -88, -80, 38, 127, -24, 60, -96, -128, 127, -124, 82, -128, -80, -80, -128, 28, 127, -36, 56, 127, 127, 6, 14, -76, 106, -128, -72,
      127, 56, -88, 127, -54, -28, -68, -56, -96, 32, 52, -60, 127, 54, 92, -80, -35, -95, -8, 36, -86, -106, -128, -105, -40, 27, -26, -33, -128, -8, -128, -63,
      26, -120, 127, -128, -52, -82, -68, -64, 48, 18, -128, -80, -128, -100, -128, 127, 40, -64, 127, 127, -2, -94, -122, -46, -42, 54, -104, -6, -28, 88, -128, 127,
      26, -120, 127, -128, -52, -82, -68, -64, 48, 18, -128, -80, -128, -100, -128, 127, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      127, -86, 127, -128, 48, -88, -80, 38, 127, -24, 60, -96, -128, 127, -124, 82, -34, 10, -118, 8, 98, -64, -66, -96, -84, -102, -82, -102, -128, 124, 2, -2,
      24, 74, -88, -50, 68, 16, 32, 127, 28, -78, 127, -2, -72, -60, 100, -124, 127, -8, -128, -52, 127, 127, -84, -48, 126, 42, 48, -74, -14, 22, -20, -128,
      -128, -30, 30, -128, 56, -128, -128, 90, 68, -128, 2, 10, 38, -110, -54, -128, 20, -28, 127, 127, 127, 127, 127, -50, 80, 12, 127, -128, 127, 127, -64, 127,
      -34, 14, -98, 82, 4, -128, 90, -120, 74, -46, 126, 127, -4, -128, 127, -128, 127, 127, -66, -12, 80, -110, -14, 30, 127, 127, 0, 110, 127, -128, 127, -8,
      28, 72, -20, -128, -128, 72, 127, 96, 32, 56, -96, -114, -100, -58, 18, 127, 127, -66, -16, -82, 14, 46, -42, -128, -96, -70, -26, -128, -56, 127, -128, 127,
      -54, -128, 24, -4, -26, 127, 88, -74, 127, 112, -128, -56, -100, -98, 54, 56, -128, -128, -26, -128, 114, -60, -44, 122, -126, -128, 127, 48, -128, 102, 112, -74,
      -26, 127, 127, 127, -128, -110, 127, -128, -128, -128, 70, -128, -128, -44, -30, -18, 40, -64, 127, 127, -2, -94, -122, -46, -42, 54, -104, -6, -28, 88, -128, 127,
      104, -72, 127, -40, -54, -12, 34, -128, -8, -36, 114, -128, 22, 127, 68, 48, 127, -66, -16, -82, 14, 46, -42, -128, -96, -70, -26, -128, -56, 127, -128, 127,
      58, 100, -88, 110, -128, 127, -84, -102, -4, -128, -32, -8, 50, 70, 28, 34, 127, -128, 127, -128, -4, 76, -116, -128, 24, 48, -62, -126, -100, -92, -128, -12,
      104, -72, 127, -40, -54, -12, 34, -128, -8, -36, 114, -128, 22, 127, 68, 48, -128, 127, 86, -100, -28, -2, 94, 16, -128, -102, 127, 127, -128, -128, -128, 26,
      -84, -128, -100, 127, -60, -22, 52, -46, -36, 100, 8, -128, -86, 0, 40, 86, -2, 14, 116, 90, 28, -52, 112, -68, -24, 127, -32, -92, -4, 44, -96, 127,
      -128, -128, 112, -128, 64, -28, 62, -64, 14, -28, -110, 127, -8, 98, -30, -128, 20, -28, 127, 127, 127, 127, 127, -50, 80, 12, 127, -128, 127, 127, -64, 127,
      -16, -64, 46, 127, -128, -20, 26, 127, -42, 4, 127, 118, -54, 127, 38, 127, -34, 10, -118, 8, 98, -64, -66, -96, -84, -102, -82, -102, -128, 124, 2, -2,
      58, 100, -88, 110, -128, 127, -84, -102, -4, -128, -32, -8, 50, 70, 28, 34, 127, -8, -128, -52, 127, 127, -84, -48, 126, 42, 48, -74, -14, 22, -20, -128,
      62, -40, 127, 127, -4, -16, 120, -114, 34, 86, -100, 127, 127, 28, 90, 110, -128, 127, 86, -100, -28, -2, 94, 16, -128, -102, 127, 127, -128, -128, -128, 26,
      22, 126, -74, -128, 110, -128, -44, 127, 127, -36, 12, -4, -36, -128, 64, -128, 126, 127, -110, -10, 12, 127, -102, -36, 127, 127, -120, 42, 8, -4, -94, -78,
      -84, -128, -99, 127, -60, -22, 52, -46, -36, 99, 8, -128, -86, 0, 40, 86, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      0, 44, 64, -128, 127, -116, 54, 50, -38, -40, 14, 28, -2, -128, -128, 36, 18, 26, 30, 98, 100, -128, 127, 127, 127, 116, 127, -82, 127, -128, 2, 92,
      -54, -128, 24, -4, -26, 127, 88, -74, 127, 112, -128, -56, -100, -98, 54, 56, 40, -64, 127, 127, -2, -94, -122, -46, -42, 54, -104, -6, -28, 88, -128, 127,
      0, -18, 6, 127, -124, 28, 127, -32, -128, 127, -128, 114, 127, -102, 114, 122, 127, -128, 127, -128, -4, 76, -116, -128, 24, 48, -62, -126, -100, -92, -128, -12,
      -84, -128, -100, 127, -60, -22, 52, -46, -36, 100, 8, -128, -86, 0, 40, 86, 126, 127, -110, -10, 12, 127, -102, -36, 127, 127, -120, 42, 8, -4, -94, -78,
      2, 12, -128, -72, -104, -100, 60, -30, -108, -38, -128, -20, -114, -128, -128, -70, -128, -48, -110, -128, -62, -28, -30, 78, -92, -128, 14, 127, -90, -72, 18, -128,
      14, -96, -36, 0, -128, 18, 126, -84, 127, 104, -128, 98, -128, -128, 94, 58, 127, -48, 12, 30, -24, -128, 127, -128, 16, 56, -118, -74, 127, -128, -94, 127,
      -26, 127, 127, 127, -128, -110, 127, -128, -128, -128, 70, -128, -128, -44, -30, -18, 127, -74, 127, -72, -2, 84, -52, -128, -128, -92, -88, -128, -26, 124, -40, 38,
      -9, -49, -128, 55, 92, 127, -80, -120, -32, 123, -123, -66, 67, -112, 29, -28, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      58, 100, -88, 110, -128, 127, -84, -102, -4, -128, -32, -8, 50, 70, 28, 34, -34, 10, -118, 8, 98, -64, -66, -96, -84, -102, -82, -102, -128, 124, 2, -2,
      -84, -128, -100, 127, -60, -22, 52, -46, -36, 100, 8, -128, -86, 0, 40, 86, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      58, 100, -88, 110, -128, 127, -84, -102, -4, -128, -32, -8, 50, 70, 28, 34, -50, 12, -10, -128, -128, -128, -128, -74, -70, 34, -66, -50, -96, -54, -62, 60,
      -64, -2, -114, 127, 127, 127, -128, -106, 96, -128, -42, -74, -6, -128, 78, -120, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      -64, -26, 98, -128, 28, 74, -128, 0, -26, -30, -90, -128, 30, 92, -128, 127, -72, -40, -88, 84, 96, -128, -128, -82, -124, -16, -84, -68, -66, -92, -2, 26,
      -94, -126, -128, -128, 127, -128, -54, 86, 10, 127, 127, 12, 104, -44, 127, 127, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      82, 3, -128, 127, -31, 65, 104, -29, 64, 11, -34, 90, 98, -101, 127, -128, 126, 127, -110, -10, 12, 127, -102, -36, 127, 127, -120, 42, 8, -4, -94, -78,
      127, 94, 26, 127, 80, 34, 58, -128, 58, -60, -4, -66, -26, -20, 127, 14, 127, -128, 6, 4, 42, 16, 127, -108, -128, -32, 24, -8, -112, 127, -128, 127,
      -84, -128, -100, 127, -60, -22, 52, -46, -36, 100, 8, -128, -86, 0, 40, 86, 127, -58, 32, -32, 90, -32, -28, -84, 127, 127, 34, -90, 127, -106, 64, 62,
      -128, 78, -128, -36, 50, -92, 76, 127, -68, 22, 120, 18, 127, -20, 48, 12, -50, 98, 127, -60, 114, 127, 127, 76, -46, 16, 18, 127, 124, -16, -6, -128,
      28, 127, 127, -34, 14, 124, -118, -74, 127, -120, 30, -90, -128, 106, -88, -56, 114, -42, -38, -12, -48, 84, 4, -122, -56, 104, 127, -28, -128, 127, -20, -14,
      100, 122, 26, -128, -86, 106, 52, 26, 78, -30, -4, 78, -70, 14, 38, -120, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      -60, 127, -128, -34, -36, 88, -30, 127, -84, 80, 42, 52, 127, 0, 24, -36, -2, 78, 46, 6, -40, -128, 127, -90, 127, 127, -128, -128, 12, -32, -128, 127,
      -34, 14, -98, 82, 4, -128, 90, -120, 74, -46, 126, 127, -4, -128, 127, -128, 114, -42, -38, -12, -48, 84, 4, -122, -56, 104, 127, -28, -128, 127, -20, -14,
      -116, 18, 127, -128, 127, 104, -28, 40, 100, 12, -82, -76, -82, 42, -94, -128, 10, 127, -128, 8, -60, 127, -104, -128, -52, 76, -58, -46, 92, -26, 68, -128,
      112, -4, -72, 44, -128, 66, 127, 18, 0, 56, -128, 8, -120, 78, 20, -4, 127, -74, 127, -72, -2, 84, -52, -128, -128, -92, -88, -128, -26, 124, -40, 38,
      10, 123, 127, -33, 30, 120, -125, -48, 108, -126, 27, -87, -112, 96, -109, -54, 36, -34, -82, 42, -68, -50, -128, 46, -128, -128, -32, -2, -128, 60, 64, -128,
      -80, 35, 127, -61, 127, 103, -128, 25, 17, -128, 46, -83, 127, 119, -128, -11, 0, -128, -36, -38, 76, 98, -34, -26, 24, 90, 48, -70, 84, -128, 64, 58,
      -41, 127, -128, -60, -32, 76, -22, 127, -115, 112, 24, 38, 127, 38, -8, -9, -104, -30, 58, -80, -122, -74, -86, 127, 30, -112, -128, -104, 38, -128, -16, -128,
      -94, 0, 2, 66, 112, -58, -24, -24, 12, -58, -66, -124, 94, -128, -62, 127, 102, 126, -128, 120, -52, 6, 82, -128, -128, 90, -82, -128, 127, 86, 26, 26,
      -26, 127, 127, 127, -128, -110, 127, -128, -128, -128, 70, -128, -128, -44, -30, -18, -2, 78, 46, 6, -40, -128, 127, -90, 127, 127, -128, -128, 12, -32, -128, 127,
      -4, -66, 70, 52, -58, 127, 88, -40, -54, 60, 124, -94, 60, 88, 32, 52, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      -54, -128, 24, -4, -26, 127, 88, -74, 127, 112, -128, -56, -100, -98, 54, 56, -36, 74, -100, -128, 127, 127, 127, 86, -84, 2, -14, -32, 82, -66, 127, -128,
      -44, 104, -128, 70, -36, 88, 127, 127, -128, 120, -10, 26, 64, 0, -62, 127, -50, 98, 127, -60, 114, 127, 127, 76, -46, 16, 18, 127, 124, -16, -6, -128,
      -94, 0, 2, 66, 111, -58, -24, -24, 12, -58, -66, -124, 93, -128, -62, 127, -2, 14, 116, 90, 28, -52, 112, -68, -24, 127, -32, -92, -4, 44, -96, 127,
      -86, 38, 127, -68, 127, 108, -128, 30, 16, -128, 52, -96, 127, 122, -128, -16, 127, -8, -128, -52, 127, 127, -84, -48, 126, 42, 48, -74, -14, 22, -20, -128,
      -64, -26, 98, -128, 28, 74, -128, 0, -26, -30, -90, -128, 30, 92, -128, 127, 34, -92, 70, 127, -26, 122, 124, -28, 127, 126, 74, -128, 40, 80, -112, 127,
      -54, -128, 24, -4, -26, 127, 88, -74, 127, 112, -128, -56, -100, -98, 54, 56, -52, 99, 127, -59, 112, 127, 127, 77, -46, 14, 18, 127, 120, -18, -4, -128,
      -22, 16, -28, -74, 30, 127, -88, 48, 28, -128, -128, 127, 24, 116, -92, -108, 127, -58, 32, -32, 90, -32, -28, -84, 127, 127, 34, -90, 127, -106, 64, 62,
      -128, -128, 112, -128, 64, -28, 62, -64, 14, -28, -110, 127, -8, 98, -30, -128, -36, 74, -100, -128, 127, 127, 127, 86, -84, 2, -14, -32, 82, -66, 127, -128,
      -128, -128, 112, -128, 64, -28, 62, -64, 14, -28, -110, 127, -8, 98, -30, -128, 78, 16, 124, -128, -50, -128, -4, 86, -128, 20, -58, -58, -44, -128, 108, 127,
      -54, -128, 24, -4, -26, 127, 88, -74, 127, 112, -128, -56, -100, -98, 54, 56, 10, 127, -128, 8, -60, 127, -104, -128, -52, 76, -58, -46, 92, -26, 68, -128,
      -94, 0, 2, 66, 112, -58, -24, -24, 12, -58, -66, -124, 94, -128, -62, 127, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      -128, 22, -118, -48, -128, -84, -70, 80, -64, -128, -126, 4, -128, -112, -90, -20, 40, -2, 118, 32, 127, 127, 84, 127, 64, 127, -4, -46, -100, -64, 96, 50,
      -94, 0, 2, 66, 112, -58, -24, -24, 12, -58, -66, -124, 94, -128, -62, 127, 127, -128, -26, 66, -44, 52, 127, -26, 54, -4, -36, -128, 82, 122, -128, 127,
      -86, 38, 127, -68, 127, 108, -128, 30, 16, -128, 52, -96, 127, 122, -128, -16, -36, 74, -100, -128, 127, 127, 127, 86, -84, 1, -14, -32, 81, -65, 127, -128,
      -22, 16, -28, -74, 30, 127, -88, 48, 28, -128, -128, 127, 24, 116, -92, -108, 52, -32, 100, 28, 92, 6, -128, -48, -100, 88, 127, -46, -128, -128, 24, -44,
      -116, 18, 127, -128, 127, 104, -28, 40, 100, 12, -82, -76, -82, 42, -94, -128, 127, -20, -10, -36, 127, -22, 126, 64, 127, 102, 114, 108, -118, 46, -128, 127,
      -128, 22, -118, -48, -128, -84, -70, 80, -64, -128, -126, 4, -128, -112, -90, -20, 126, 127, -110, -10, 12, 127, -102, -36, 127, 127, -120, 42, 8, -4, -94, -78,
      -128, -128, 112, -128, 64, -28, 62, -64, 14, -28, -110, 127, -8, 98, -30, -128, -92, 50, -104, 52, 0, 10, 104, -42, -112, -112, 18, 36, 42, -38, 18, 127,
      -84, -128, -100, 127, -60, -22, 52, -46, -36, 100, 8, -128, -86, 0, 40, 86, 14, 94, -112, 127, 12, -16, -128, -128, 94, 127, -56, -28, -4, -16, -128, 60,
      -42, 38, -2, -126, 80, -124, -50, 58, -78, -128, 88, -60, -66, -112, -66, -8, -70, -88, 127, 127, 108, -128, -30, 12, 28, 8, 127, -44, 14, -124, -124, 34,
      -86, 38, 127, -68, 127, 108, -128, 30, 16, -128, 52, -96, 127, 122, -128, -16, 74, 127, -84, 127, 8, -68, -64, 30, -128, 10, 26, -128, -86, -56, -110, 127,
      76, -128, 54, 127, 127, -10, 74, -128, 8, 127, 50, 16, -16, -120, 70, -8, -50, 98, 127, -60, 114, 127, 127, 76, -46, 16, 18, 127, 124, -16, -6, -128};

  run_qordered_longformer_attention_op_test(
      gold_result,
      inputq,
      weightq,
      bias,
      attention_mask,
      global_weightq,
      global_bias,
      global_attention_mask,
      scale_input,
      scale_weight,
      scale_bias,
      scale_qkv_gemm,
      scale_global_weight,
      scale_global_gemm,
      scale_output,
      batch_size,
      sequence_len,
      num_heads,
      head_size,
      window,
      input_hidden_size);
}

}  // namespace test
}  // namespace onnxruntime

#endif  // CUDA_VERSION

#endif  // USE_CUDA
